diff --git a/src/vanilla.ts b/src/vanilla.ts
index 625e10e9..bff2b11f 100644
--- a/src/vanilla.ts
+++ b/src/vanilla.ts
@@ -168,7 +168,9 @@ const buildProxyFunction = (
       readonly [ProxyState, RemoveListener?]
     >()
     const addPropListener = (prop: string | symbol, propValue: unknown) => {
-      const propProxyState = proxyStateMap.get(propValue as object)
+      const propProxyState =
+        !refSet.has(propValue as object) &&
+        proxyStateMap.get(propValue as object)
       if (propProxyState) {
         if (
           import.meta.env?.MODE !== 'production' &&