Merge branch 'talzacc/ccm_maps_fix' of https://github.com/microsoft/g…

…arnet into talzacc/ccm_maps_fix
microsoft · Dec 7, 2024 · e68d00c · e68d00c
2 parents 7873157 + f9ad364
commit e68d00c
Show file tree

Hide file tree

Showing 33 changed files with 680 additions and 129 deletions.
diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml
@@ -1,4 +1,7 @@
+name: Garnet Helm Chart
+
 on:
+  workflow_dispatch: # allow manual run
   push:
     branches:
       - main
@@ -33,12 +36,12 @@ jobs:
 
       - name: Set helm chart appVersion
         run: |
-          sed -ie 's#Version.props#'"${VERSION_PROPS}"'#g' charts/garnet/Chart.yaml
+          sed -i -e 's#Version.props#"'${VERSION_PROPS}'"#g' charts/garnet/Chart.yaml
 
       - name: Helm lint and package
         run: |
           mkdir .cr-release-packages
-          for chart in $(find charts -depth 1 -type d); do
+          for chart in $(find charts -mindepth 1 -maxdepth 1 -type d); do
             if [ -z "${chart:-}" ]; then
               break
             fi

diff --git a/charts/garnet/Chart.yaml b/charts/garnet/Chart.yaml
@@ -1,20 +1,27 @@
 apiVersion: v2
 name: garnet
-description: A Helm chart for Microsoft garnet
+description: A Helm chart for Microsoft Garnet
 type: application
-version: 0.1.2
+version: 0.1.3
 appVersion: Version.props
 home: https://github.com/microsoft/garnet
 icon: https://avatars.githubusercontent.com/u/6154722?s=200&v=4
 
 keywords:
-  - garnet
-  - database
-  - key
-  - value
-  - in-memory
+- scalable
+- key-value
+- cache
+- cluster
+- store
+- remote
+- persistent
+- concurrent
+- low-latency
+- cache-storage
+- hash-table
+- larger-the-memory
 
 sources:
-  - https://github.com/microsoft/garnet.git
+- https://github.com/microsoft/garnet.git
 
 maintainers: []
diff --git a/charts/garnet/README.md b/charts/garnet/README.md
@@ -1,8 +1,8 @@
 # garnet
 
-![Version: 0.1.2](https://img.shields.io/badge/Version-0.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.46](https://img.shields.io/badge/AppVersion-1.0.46-informational?style=flat-square)
+![Version: 0.1.3](https://img.shields.io/badge/Version-0.1.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.46](https://img.shields.io/badge/AppVersion-1.0.46-informational?style=flat-square)
 
-A Helm chart for Microsoft garnet
+A Helm chart for Microsoft Garnet
 
 **Homepage:** <https://github.com/microsoft/garnet>
 
@@ -15,7 +15,7 @@ A Helm chart for Microsoft garnet
 [Helm](https://helm.sh) must be installed to use the charts. Please refer to
 Helm's [documentation](https://helm.sh/docs) to get started.
 
-To install the garnet chart (using an OCI-based registry):
+To install the Garnet chart (using an OCI-based registry):
 
 ```sh
 helm upgrade --install garnet oci://ghcr.io/microsoft/helm-charts/garnet

diff --git a/charts/garnet/README.md.gotmpl b/charts/garnet/README.md.gotmpl
@@ -16,7 +16,7 @@
 [Helm](https://helm.sh) must be installed to use the charts. Please refer to
 Helm's [documentation](https://helm.sh/docs) to get started.
 
-To install the garnet chart (using an OCI-based registry):
+To install the Garnet chart (using an OCI-based registry):
 
 ```sh
 helm upgrade --install garnet oci://ghcr.io/microsoft/helm-charts/garnet

diff --git a/libs/cluster/Server/ClusterManagerSlotState.cs b/libs/cluster/Server/ClusterManagerSlotState.cs
@@ -383,6 +383,20 @@ public bool TryPrepareSlotForOwnershipChange(int slot, string nodeid, out ReadOn
                 FlushConfig();
                 return true;
             }
+            else
+            {
+                while (true)
+                {
+                    current = currentConfig;
+                    workerId = current.GetWorkerIdFromNodeId(nodeid);
+                    var newConfig = current.UpdateSlotState(slot, workerId, SlotState.STABLE);
+
+                    if (Interlocked.CompareExchange(ref currentConfig, newConfig, current) == current)
+                        break;
+                }
+                FlushConfig();
+                logger?.LogTrace("[Processed] SetSlot {slot} FORCED TO {nodeId}", slot, nodeid);
+            }
             return true;
         }
 

diff --git a/libs/cluster/Server/ClusterManagerWorkerState.cs b/libs/cluster/Server/ClusterManagerWorkerState.cs
@@ -198,6 +198,9 @@ public bool TryAddReplica(string nodeid, bool force, out ReadOnlySpan<byte> erro
                 var newConfig = currentConfig.MakeReplicaOf(nodeid).BumpLocalNodeConfigEpoch();
                 if (Interlocked.CompareExchange(ref currentConfig, newConfig, current) == current)
                     break;
+
+                // If we reach here then we failed to update config so we need to suspend recovery and retry to update the config
+                clusterProvider.replicationManager.SuspendRecovery();
             }
             FlushConfig();
             return true;

diff --git a/libs/cluster/Server/Failover/ReplicaFailoverSession.cs b/libs/cluster/Server/Failover/ReplicaFailoverSession.cs
@@ -167,16 +167,16 @@ private bool TakeOverAsPrimary()
             // Take over as primary and inform old primary
             status = FailoverStatus.TAKING_OVER_AS_PRIMARY;
 
-            // Make replica syncing unavailable by setting recovery flag
-            if (!clusterProvider.replicationManager.StartRecovery())
-            {
-                logger?.LogWarning($"{nameof(TakeOverAsPrimary)}: {{logMessage}}", Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_CANNOT_ACQUIRE_RECOVERY_LOCK));
-                return false;
-            }
-            _ = clusterProvider.BumpAndWaitForEpochTransition();
-
             try
             {
+                // Make replica syncing unavailable by setting recovery flag
+                if (!clusterProvider.replicationManager.StartRecovery())
+                {
+                    logger?.LogWarning($"{nameof(TakeOverAsPrimary)}: {{logMessage}}", Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_CANNOT_ACQUIRE_RECOVERY_LOCK));
+                    return false;
+                }
+                _ = clusterProvider.BumpAndWaitForEpochTransition();
+
                 // Take over slots from old primary
                 if (!clusterProvider.clusterManager.TryTakeOverForPrimary())
                 {

diff --git a/libs/cluster/Server/Replication/ReplicationManager.cs b/libs/cluster/Server/Replication/ReplicationManager.cs
@@ -110,7 +110,7 @@ public ReplicationManager(ClusterProvider clusterProvider, ILogger logger = null
                 clusterProvider.GetReplicationLogCheckpointManager(StoreType.Object).checkpointVersionShift = CheckpointVersionShift;
 
             // If this node starts as replica, it cannot serve requests until it is connected to primary
-            if (clusterProvider.clusterManager.CurrentConfig.LocalNodeRole == NodeRole.REPLICA && !StartRecovery())
+            if (clusterProvider.clusterManager.CurrentConfig.LocalNodeRole == NodeRole.REPLICA && clusterProvider.serverOptions.Recover && !StartRecovery())
                 throw new Exception(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_CANNOT_ACQUIRE_RECOVERY_LOCK));
 
             checkpointStore = new CheckpointStore(storeWrapper, clusterProvider, true, logger);
@@ -277,7 +277,7 @@ public void Start()
 
             var localNodeRole = current.LocalNodeRole;
             var replicaOfNodeId = current.LocalNodePrimaryId;
-            if (localNodeRole == NodeRole.REPLICA && replicaOfNodeId != null)
+            if (localNodeRole == NodeRole.REPLICA && clusterProvider.serverOptions.Recover && replicaOfNodeId != null)
             {
                 // At initialization of ReplicationManager, this node has been put into recovery mode
                 if (!TryReplicateFromPrimary(out var errorMessage))

diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json
@@ -684,6 +684,37 @@
       }
     ]
   },
+  {
+    "Command": "BRPOPLPUSH",
+    "Name": "BRPOPLPUSH",
+    "Summary": "Pops an element from a list, pushes it to another list and returns it. Block until an element is available otherwise. Deletes the list if the last element was popped.",
+    "Group": "List",
+    "Complexity": "O(1)",
+    "DocFlags": "Deprecated",
+    "ReplacedBy": "\u0060BLMOVE\u0060 with the \u0060RIGHT\u0060 and \u0060LEFT\u0060 arguments",
+    "Arguments": [
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "SOURCE",
+        "DisplayText": "source",
+        "Type": "Key",
+        "KeySpecIndex": 0
+      },
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "DESTINATION",
+        "DisplayText": "destination",
+        "Type": "Key",
+        "KeySpecIndex": 1
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "TIMEOUT",
+        "DisplayText": "timeout",
+        "Type": "Double"
+      }
+    ]
+  },
   {
     "Command": "CLIENT",
     "Name": "CLIENT",
@@ -6093,6 +6124,57 @@
       }
     ]
   },
+  {
+    "Command": "ZREVRANGEBYLEX",
+    "Name": "ZREVRANGEBYLEX",
+    "Summary": "Returns members in a sorted set within a lexicographical range in reverse order.",
+    "Group": "SortedSet",
+    "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).",
+    "DocFlags": "Deprecated",
+    "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 and \u0060BYLEX\u0060 arguments",
+    "Arguments": [
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "KEY",
+        "DisplayText": "key",
+        "Type": "Key",
+        "KeySpecIndex": 0
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "MAX",
+        "DisplayText": "max",
+        "Type": "String"
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "MIN",
+        "DisplayText": "min",
+        "Type": "String"
+      },
+      {
+        "TypeDiscriminator": "RespCommandContainerArgument",
+        "Name": "LIMIT",
+        "Type": "Block",
+        "Token": "LIMIT",
+        "ArgumentFlags": "Optional",
+        "Arguments": [
+          {
+            "TypeDiscriminator": "RespCommandBasicArgument",
+            "Name": "OFFSET",
+            "DisplayText": "offset",
+            "Type": "Integer"
+          },
+          {
+            "TypeDiscriminator": "RespCommandBasicArgument",
+            "Name": "COUNT",
+            "DisplayText": "count",
+            "Type": "Integer"
+          }
+        ]
+      }
+    ]
+  },
   {
     "Command": "ZREVRANGEBYSCORE",
     "Name": "ZREVRANGEBYSCORE",

diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json
@@ -355,6 +355,44 @@
       }
     ]
   },
+  {
+    "Command": "BRPOPLPUSH",
+    "Name": "BRPOPLPUSH",
+    "Arity": 4,
+    "Flags": "Blocking, DenyOom, Write",
+    "FirstKey": 1,
+    "LastKey": 2,
+    "Step": 1,
+    "AclCategories": "Blocking, List, Slow, Write",
+    "KeySpecifications": [
+      {
+        "BeginSearch": {
+          "TypeDiscriminator": "BeginSearchIndex",
+          "Index": 1
+        },
+        "FindKeys": {
+          "TypeDiscriminator": "FindKeysRange",
+          "LastKey": 0,
+          "KeyStep": 1,
+          "Limit": 0
+        },
+        "Flags": "RW, Access, Delete"
+      },
+      {
+        "BeginSearch": {
+          "TypeDiscriminator": "BeginSearchIndex",
+          "Index": 2
+        },
+        "FindKeys": {
+          "TypeDiscriminator": "FindKeysRange",
+          "LastKey": 0,
+          "KeyStep": 1,
+          "Limit": 0
+        },
+        "Flags": "RW, Insert"
+      }
+    ]
+  },
   {
     "Command": "CLIENT",
     "Name": "CLIENT",
@@ -4514,6 +4552,31 @@
       }
     ]
   },
+  {
+    "Command": "ZREVRANGEBYLEX",
+    "Name": "ZREVRANGEBYLEX",
+    "Arity": -4,
+    "Flags": "ReadOnly",
+    "FirstKey": 1,
+    "LastKey": 1,
+    "Step": 1,
+    "AclCategories": "Read, SortedSet, Slow",
+    "KeySpecifications": [
+      {
+        "BeginSearch": {
+          "TypeDiscriminator": "BeginSearchIndex",
+          "Index": 1
+        },
+        "FindKeys": {
+          "TypeDiscriminator": "FindKeysRange",
+          "LastKey": 0,
+          "KeyStep": 1,
+          "Limit": 0
+        },
+        "Flags": "RO, Access"
+      }
+    ]
+  },
   {
     "Command": "ZREVRANGEBYSCORE",
     "Name": "ZREVRANGEBYSCORE",

diff --git a/libs/server/Objects/SortedSet/SortedSetObject.cs b/libs/server/Objects/SortedSet/SortedSetObject.cs
@@ -35,6 +35,7 @@ public enum SortedSetOperation : byte
         GEOSEARCH,
         GEOSEARCHSTORE,
         ZREVRANGE,
+        ZREVRANGEBYLEX,
         ZREVRANGEBYSCORE,
         ZREVRANK,
         ZREMRANGEBYLEX,
@@ -280,6 +281,7 @@ public override unsafe bool Operate(ref ObjectInput input, ref SpanByteAndMemory
                     case SortedSetOperation.ZREVRANGE:
                         SortedSetRange(ref input, ref output);
                         break;
+                    case SortedSetOperation.ZREVRANGEBYLEX:
                     case SortedSetOperation.ZREVRANGEBYSCORE:
                         SortedSetRange(ref input, ref output);
                         break;

diff --git a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs
@@ -445,6 +445,10 @@ private void SortedSetRange(ref ObjectInput input, ref SpanByteAndMemory output)
                         options.ByScore = true;
                         options.Reverse = true;
                         break;
+                    case SortedSetOperation.ZREVRANGEBYLEX:
+                        options.ByLex = true;
+                        options.Reverse = true;
+                        break;
                 }
 
                 if (count > 2)

diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs
@@ -111,6 +111,10 @@ static partial class CmdStrings
         public static ReadOnlySpan<byte> STOREDIST => "STOREDIST"u8;
         public static ReadOnlySpan<byte> WITHDIST => "WITHDIST"u8;
         public static ReadOnlySpan<byte> WITHHASH => "WITHHASH"u8;
+        public static ReadOnlySpan<byte> RIGHT => "RIGHT"u8;
+        public static ReadOnlySpan<byte> LEFT => "LEFT"u8;
+        public static ReadOnlySpan<byte> BYLEX => "BYLEX"u8;
+        public static ReadOnlySpan<byte> REV => "REV"u8;
 
         /// <summary>
         /// Response strings