diff --git a/web/src/lib/components/elements/buttons/circle-icon-button.svelte b/web/src/lib/components/elements/buttons/circle-icon-button.svelte
index 155062c1e26f4..071a645000007 100644
--- a/web/src/lib/components/elements/buttons/circle-icon-button.svelte
+++ b/web/src/lib/components/elements/buttons/circle-icon-button.svelte
@@ -1,5 +1,5 @@
 <script lang="ts" module>
-  export type Color = 'transparent' | 'light' | 'dark' | 'red' | 'gray' | 'primary' | 'opaque' | 'alert';
+  export type Color = 'transparent' | 'light' | 'dark' | 'red' | 'gray' | 'primary' | 'opaque' | 'alert' | 'neutral';
   export type Padding = '1' | '2' | '3';
 </script>
 
@@ -68,6 +68,8 @@
     dark: 'bg-[#202123] hover:bg-[#d3d3d3]',
     alert: 'text-[#ff0000] hover:text-white',
     gray: 'bg-[#d3d3d3] hover:bg-[#e2e7e9] text-immich-dark-gray hover:text-black',
+    neutral:
+      'dark:bg-immich-dark-gray dark:text-gray-300 hover:dark:bg-immich-dark-gray/50 hover:dark:text-gray-300 bg-gray-200 text-gray-700 hover:bg-gray-300',
     primary:
       'bg-immich-primary dark:bg-immich-dark-primary hover:bg-immich-primary/75 hover:dark:bg-immich-dark-primary/80 text-white dark:text-immich-dark-gray',
   };
diff --git a/web/src/lib/components/faces-page/assign-face-side-panel.svelte b/web/src/lib/components/faces-page/assign-face-side-panel.svelte
index f2ee970b3282c..fe6a454307f3c 100644
--- a/web/src/lib/components/faces-page/assign-face-side-panel.svelte
+++ b/web/src/lib/components/faces-page/assign-face-side-panel.svelte
@@ -15,7 +15,6 @@
   import { t } from 'svelte-i18n';
   import { handleError } from '$lib/utils/handle-error';
   import { onMount } from 'svelte';
-  import SettingSwitch from '$lib/components/shared-components/settings/setting-switch.svelte';
 
   interface Props {
     editedFace: AssetFaceResponseDto;
@@ -35,10 +34,7 @@
   async function loadPeople() {
     const timeout = setTimeout(() => (isShowLoadingPeople = true), timeBeforeShowLoadingSpinner);
     try {
-      const { people } = await getAllPeople({
-        withHidden: true,
-        closestAssetId: sortFaces ? editedFace.id : undefined,
-      });
+      const { people } = await getAllPeople({ withHidden: true, closestAssetId: editedFace.id });
       allPeople = people;
     } catch (error) {
       handleError(error, $t('errors.cant_get_faces'));
@@ -56,7 +52,6 @@
   let searchedPeople: PersonResponseDto[] = $state([]);
   let searchFaces = $state(false);
   let searchName = $state('');
-  let sortFaces = $state(true);
 
   let showPeople = $derived(searchName ? searchedPeople : allPeople.filter((person) => !person.isHidden));
 
@@ -122,16 +117,6 @@
     {/if}
   </div>
   <div class="px-4 py-4 text-sm">
-    <div class="flex w-full justify-center">
-      <SettingSwitch
-        bind:checked={sortFaces}
-        onToggle={async (checked) => {
-          sortFaces = checked;
-          await loadPeople();
-        }}
-        title={$t('sort_people_by_similarity')}
-      />
-    </div>
     <h2 class="mb-8 mt-4 uppercase">{$t('all_people')}</h2>
     {#if isShowLoadingPeople}
       <div class="flex w-full justify-center">
diff --git a/web/src/lib/components/faces-page/merge-face-selector.svelte b/web/src/lib/components/faces-page/merge-face-selector.svelte
index 673ae9b2652c2..5d0777b233894 100644
--- a/web/src/lib/components/faces-page/merge-face-selector.svelte
+++ b/web/src/lib/components/faces-page/merge-face-selector.svelte
@@ -34,9 +34,7 @@
   let hasSelection = $derived(selectedPeople.length > 0);
   let peopleToNotShow = $derived([...selectedPeople, person]);
 
-  let sortFaces = $state(true);
-
-  const handleSearch = async () => {
+  const handleSearch = async (sortFaces: boolean = false) => {
     const data = await getAllPeople({ withHidden: false, closestPersonId: sortFaces ? person.id : undefined });
     people = data.people;
   };
@@ -153,7 +151,7 @@
           <FaceThumbnail {person} border circle selectable={false} thumbnailSize={180} />
         </div>
       </div>
-      <PeopleList {people} {peopleToNotShow} {screenHeight} {onSelect} bind:sortFaces {handleSearch} />
+      <PeopleList {people} {peopleToNotShow} {screenHeight} {onSelect} {handleSearch} />
     </section>
   </section>
 </section>
diff --git a/web/src/lib/components/faces-page/people-list.svelte b/web/src/lib/components/faces-page/people-list.svelte
index ab832739c9d68..1c1eee39ec58a 100644
--- a/web/src/lib/components/faces-page/people-list.svelte
+++ b/web/src/lib/components/faces-page/people-list.svelte
@@ -3,21 +3,20 @@
   import FaceThumbnail from './face-thumbnail.svelte';
   import SearchPeople from '$lib/components/faces-page/people-search.svelte';
   import { t } from 'svelte-i18n';
-  import SettingSwitch from '$lib/components/shared-components/settings/setting-switch.svelte';
+  import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';
+  import { mdiSwapVertical } from '@mdi/js';
 
   interface Props {
-    sortFaces?: boolean;
     screenHeight: number;
     people: PersonResponseDto[];
     peopleToNotShow: PersonResponseDto[];
     onSelect: (person: PersonResponseDto) => void;
-    handleSearch?: () => void;
+    handleSearch?: (sortFaces: boolean) => void;
   }
 
-  let { sortFaces = $bindable(true), screenHeight, people, peopleToNotShow, onSelect, handleSearch }: Props = $props();
-
+  let { screenHeight, people, peopleToNotShow, onSelect, handleSearch }: Props = $props();
   let searchedPeopleLocal: PersonResponseDto[] = $state([]);
-
+  let sortBySimilarirty = $state(false);
   let name = $state('');
 
   const showPeople = $derived(
@@ -27,23 +26,26 @@
   );
 </script>
 
-<div class=" w-40 sm:w-48 md:w-96 h-14 mb-8">
-  <SearchPeople type="searchBar" placeholder={$t('search_people')} bind:searchName={name} bind:searchedPeopleLocal />
-</div>
-{#if handleSearch}
-  <div class=" w-40 sm:w-48 md:w-96 mb-8">
-    <SettingSwitch
-      onToggle={(checked) => {
-        sortFaces = checked;
-        handleSearch();
+<div class="w-40 sm:w-48 md:w-full h-14 flex gap-4 place-items-center">
+  <div class="md:w-96">
+    <SearchPeople type="searchBar" placeholder={$t('search_people')} bind:searchName={name} bind:searchedPeopleLocal />
+  </div>
+
+  {#if handleSearch}
+    <CircleIconButton
+      icon={mdiSwapVertical}
+      onclick={() => {
+        sortBySimilarirty = !sortBySimilarirty;
+        handleSearch(sortBySimilarirty);
       }}
-      bind:checked={sortFaces}
+      color="neutral"
       title={$t('sort_people_by_similarity')}
-    ></SettingSwitch>
-  </div>
-{/if}
+    ></CircleIconButton>
+  {/if}
+</div>
+
 <div
-  class="immich-scrollbar overflow-y-auto rounded-3xl bg-gray-200 p-10 dark:bg-immich-dark-gray"
+  class="immich-scrollbar overflow-y-auto rounded-3xl bg-gray-200 p-10 dark:bg-immich-dark-gray mt-6"
   style:max-height={screenHeight - 400 + 'px'}
 >
   <div class="grid-col-2 grid gap-8 md:grid-cols-3 lg:grid-cols-6 xl:grid-cols-8 2xl:grid-cols-10">