Skip to content

Commit

Permalink
Merge pull request #329 from UpstreetAI/dev
Browse files Browse the repository at this point in the history
Merge Dev
  • Loading branch information
mavisakalyan authored Oct 16, 2024
2 parents 7a96fa6 + 2882522 commit 0106e2c
Show file tree
Hide file tree
Showing 71 changed files with 2,331 additions and 3,554 deletions.
3 changes: 0 additions & 3 deletions apps/chat/app/layout.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,12 @@ export default function RootLayout({ children }: RootLayoutProps) {
enableSystem
disableTransitionOnChange
>
{/*<Suspense fallback={<div>Loading...</div>}>*/}
<div className="flex flex-col min-h-screen">
<Body>
<Header />
{children}
</Body>
</div>
{/*</Suspense>*/}

<TailwindIndicator/>
<Footer />
</Providers>
Expand Down
5 changes: 1 addition & 4 deletions apps/chat/components/chat/chat-menu.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,14 @@ export function ChatMenu({ players, roomName }: ChatMenuProps) {
const { toggleRightSidebar, isLeftSidebarOpen, isRightSidebarOpen } = useSidebar();

const pathname = usePathname();
if (pathname.startsWith('/new')) {
//return null;
}

return (
<div
className={`absolute z-[10] bg-gray-300 left-0 w-full ease-in-out duration-300 animate-in border-b ${isLeftSidebarOpen ? 'lg:pl-[250px] xl:pl-[300px]' : ''
} ${isRightSidebarOpen ? 'lg:pr-[250px] xl:pr-[300px]' : ''}`}
>

<div className='absolute z-[100] left-3 md:left-4 top-1/2 transform -translate-y-1/2 mt-1'>
<div className={`${pathname.startsWith('/new') && 'hidden'} absolute z-[100] left-3 md:left-4 top-1/2 transform -translate-y-1/2 mt-1`}>
<IconButton href={"/"} icon={'BackArrow'} />
</div>

Expand Down
63 changes: 33 additions & 30 deletions apps/chat/components/chat/chat.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import { PaymentItem, SubscriptionProps } from 'react-agents/types';
import { createSession } from 'react-agents/util/stripe-utils.mjs';
import { webbrowserActionsToText } from 'react-agents/util/browser-action-utils.mjs';
import { currencies, intervals } from 'react-agents/constants.mjs';
import { IconButton } from 'ucom';
// import { IconButton } from 'ucom';
import { ChatMenu } from './chat-menu';
import { useLoading } from '@/lib/client/hooks/use-loading';

Expand Down Expand Up @@ -143,36 +143,39 @@ export function Chat({ className, /* user, missingKeys, */ room, onConnect }: Ch
className={`relative group w-full duration-300 text-gray-900 ease-in-out animate-in ${isLeftSidebarOpen ? "lg:pl-[250px] xl:pl-[300px]" : ""} ${isRightSidebarOpen ? "lg:pr-[250px] xl:pr-[300px]" : ""} `}
>

<ChatMenu players={players} roomName={roomName} />

<div className='h-screen overflow-auto' ref={scrollRef}>
<div
className={cn('pb-[200px] pt-20 md:pt-24', className)}
ref={messagesRef}
>
{room ? (
messages.length ? (
<ChatList messages={messages} />
) : (
null
)
) : <EmptyScreen />}

<div className="relative mx-auto max-w-2xl px-4">
{isAgentLoading && "Loading agent..."}
{room && (
<>
<ChatMenu players={players} roomName={roomName} />

<div className='h-screen overflow-auto' ref={scrollRef}>
<div
className={cn('pb-[200px] pt-20 md:pt-24', className)}
ref={messagesRef}
>
{messages.length ? (
<ChatList messages={messages} />
) : (
null
)}

<div className="relative mx-auto max-w-2xl px-4">
{isAgentLoading && "Loading agent..."}
</div>

<div className="w-full h-px" ref={visibilityRef} />
</div>
</div>

<div className="w-full h-px" ref={visibilityRef} />
</div>
</div>
<ChatPanel
input={input}
setInput={setInput}
isAtBottom={isAtBottom}
scrollToBottom={scrollToBottom}
room={room}
messages={messages}
/>
<ChatPanel
input={input}
setInput={setInput}
isAtBottom={isAtBottom}
scrollToBottom={scrollToBottom}
room={room}
messages={messages}
/>

</>
)}
</div>
)
}
Expand Down
131 changes: 118 additions & 13 deletions apps/chat/components/prompt-form.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,22 @@ import * as React from 'react'
import Textarea from 'react-textarea-autosize'

import { Button } from '@/components/ui/button'
import { IconPlus, IconImage, IconDocument, IconAudio, IconCapture, IconTriangleDown, IconTriangleSmallDown, IconUpstreet } from '@/components/ui/icons'
import { IconPlus, IconImage, IconDocument, IconAudio, IconVideo, IconCapture, IconTriangleDown, IconTriangleSmallDown, IconUpstreet } from '@/components/ui/icons'
import {
Tooltip,
TooltipContent,
TooltipTrigger
} from '@/components/ui/tooltip'
import { useMultiplayerActions } from '@/components/ui/multiplayer-actions'
import { cn } from '@/lib/utils'
import type { PlayableAudioStream } from 'react-agents/types';
import type {
PlayableAudioStream,
PlayableVideoStream,
} from 'react-agents/types';
import { shuffle } from 'react-agents/util/util.mjs';
import { Icon } from 'ucom';
import { createPcmF32MicrophoneSource } from '@upstreet/multiplayer/public/audio/audio-client.mjs';
import { createVideoSource } from '@upstreet/multiplayer/public/video/video-client.mjs';
import { ensureAudioContext } from '@/lib/audio/audio-context-output';
// import { consoleImageWidth } from 'react-agents/constants.mjs'

Expand All @@ -28,9 +32,11 @@ export function PromptForm({
}) {
const [mediaPickerOpen, setMediaPickerOpen] = React.useState(false);
const inputRef = React.useRef<HTMLTextAreaElement>(null)
const { connected, playersMap, typingMap, sendNudgeMessage, sendChatMessage, sendMediaMessage, addAudioSource, removeAudioSource } = useMultiplayerActions()
const { connected, playersMap, typingMap, sendNudgeMessage, sendChatMessage, sendMediaMessage, addAudioSource, removeAudioSource, addVideoSource, removeVideoSource } = useMultiplayerActions()
const [typing, setTyping] = React.useState('');
const [microphoneSource, setMicrophoneSource] = React.useState<any>(null);
const [cameraSource, setCameraSource] = React.useState<any>(null);
const [screenSource, setScreenSource] = React.useState<any>(null);

React.useEffect(() => {
// typing
Expand Down Expand Up @@ -166,9 +172,6 @@ export function PromptForm({
mediaStream,
audioContext,
});
// microphoneSource.output.addEventListener('data', (e: any) => {
// console.log('got data', e.data);
// });
setMicrophoneSource(microphoneSource);

const audioStream = new ReadableStream({
Expand All @@ -182,7 +185,7 @@ export function PromptForm({
},
}) as PlayableAudioStream;
audioStream.id = crypto.randomUUID();
audioStream.type = 'audio/pcm-f32';
audioStream.type = 'audio/pcm-f32-48000';
audioStream.disposition = 'text';

(async () => {
Expand All @@ -205,16 +208,118 @@ export function PromptForm({
<IconAudio className="mr-2" />
<div>Audio</div>
</Button>
{/* <Button
<Button
variant="secondary"
className="flex justify-start relative rounded bg-background mx-2 p-2 overflow-hidden"
onClick={() => {
console.log('click capture');
className={cn("flex justify-start relative rounded bg-background mx-2 p-2 overflow-hidden")}
onClick={async () => {
if (!cameraSource) {
// list the available mics
const devices = await navigator.mediaDevices.enumerateDevices();
// console.log('got devices', devices);
const videoInputDevices = devices.filter((device) => device.kind === 'videoinput');
// const micAudioInputDevices = audioInputDevices.filter((device) => /mic/i.test(device.label));
// const otherAudioInputDevices = audioInputDevices.filter((device) => !/mic/i.test(device.label));
// const audioInputDevice = micAudioInputDevices[0] || otherAudioInputDevices[0] || null;
const videoInputDevice = videoInputDevices[0] || null;
if (videoInputDevice) {
const mediaStream = await navigator.mediaDevices.getUserMedia({
video: {
deviceId: videoInputDevice.deviceId,
},
});
// console.log('got media stream', mediaStream);
const cameraSource = createVideoSource({
mediaStream,
});
setCameraSource(cameraSource);

const videoStream = new ReadableStream({
start(controller) {
cameraSource.output.addEventListener('data', (e: any) => {
controller.enqueue(e.data);
});
cameraSource.output.addEventListener('end', (e: any) => {
controller.close();
});
},
}) as PlayableVideoStream;
videoStream.id = crypto.randomUUID();
videoStream.type = 'image/webp';
videoStream.disposition = 'text';

(async () => {
console.log('start streaming');
const {
waitForFinish,
} = addVideoSource(videoStream);
await waitForFinish();
removeVideoSource(videoStream);
})();
} else {
console.warn('no video input device found');
}
} else {
cameraSource.close();
setCameraSource(null);
}
}}
>
<IconVideo className="mr-2" />
<div>Camera</div>
</Button>
<Button
variant="secondary"
className={cn("flex justify-start relative rounded bg-background mx-2 p-2 overflow-hidden")}
onClick={async () => {
if (!screenSource) {
// list the available mics
const devices = await navigator.mediaDevices.enumerateDevices();
// console.log('got devices', devices);
// const videoInputDevices = devices.filter((device) => device.kind === 'videoinput');
const mediaStream = await navigator.mediaDevices.getDisplayMedia({
// video: {
// cursor: "always", // or "motion" or "never"
// },
video: true,
audio: false, // Set to true if you also want to capture audio
});
// console.log('got media stream', mediaStream);
const screenSource = createVideoSource({
mediaStream,
});
setScreenSource(screenSource);

const videoStream = new ReadableStream({
start(controller) {
screenSource.output.addEventListener('data', (e: any) => {
controller.enqueue(e.data);
});
screenSource.output.addEventListener('end', (e: any) => {
controller.close();
});
},
}) as PlayableVideoStream;
videoStream.id = crypto.randomUUID();
videoStream.type = 'image/webp';
videoStream.disposition = 'text';

(async () => {
console.log('start streaming');
const {
waitForFinish,
} = addVideoSource(videoStream);
await waitForFinish();
removeVideoSource(videoStream);
})();
} else {
screenSource.close();
setScreenSource(null);
}
}}
>
<IconCapture className="mr-2" />
<div>Capture</div>
</Button> */}
<div>Screen</div>
</Button>
</div>
)}
<Tooltip>
Expand Down
4 changes: 2 additions & 2 deletions apps/chat/components/ui/Header/Header.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ export async function Header() {
return null;
}
})();

const credits = await getCredits(user.id);
const credits = user ? await getCredits(user.id) : null;

return (
<HeaderNavigation user={user} credits={credits} />
Expand Down
2 changes: 1 addition & 1 deletion apps/chat/components/ui/icons.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -676,7 +676,7 @@ function IconVideo({ className, ...props }: React.ComponentProps<'svg'>) {
className={cn('size-4', className)}
{...props}
>
<path d="m269.53 495.71h409.22c14.625 0 27.938-6 37.641-15.656 9.7031-9.7031 15.656-23.016 15.656-37.641s-6-27.938-15.656-37.641c-9.7031-9.7031-23.016-15.656-37.641-15.656h-409.22c-14.625 0-27.984 6-37.641 15.656-9.7031 9.7031-15.703 23.016-15.703 37.641 0 14.672 6 27.984 15.703 37.641 9.7031 9.7031 23.016 15.656 37.641 15.656zm-257.34-184.74v578.11c0 40.219 16.453 76.734 42.891 103.22 26.438 26.438 63 42.891 103.22 42.891h631.64c40.219 0 76.734-16.406 103.22-42.891 26.438-26.438 42.891-63 42.891-103.22v-40.312l96.141 43.5c17.391 7.875 35.531 11.109 53.438 9.7969 18-1.3125 35.531-7.2188 51.609-17.578 16.078-10.359 28.641-23.906 37.312-39.797 8.6719-15.891 13.266-33.797 13.266-52.828v-383.68c0-19.031-4.5938-36.891-13.266-52.828-8.625-15.891-21.234-29.391-37.312-39.75-16.078-10.359-33.562-16.219-51.609-17.625-17.906-1.3125-36 1.9219-53.438 9.7969l-96.141 43.5v-40.312c0-40.219-16.406-76.734-42.891-103.22-26.438-26.438-63-42.891-103.22-42.891h-631.64c-40.219 0-76.734 16.406-103.22 42.891-26.438 26.438-42.891 63-42.891 103.22zm666.56 239.81h-409.22c-29.859 0-56.953-12.188-76.594-31.828s-31.828-46.734-31.828-76.594 12.188-56.953 31.828-76.594 46.734-31.828 76.594-31.828h409.22c29.859 0 56.953 12.188 76.594 31.828s31.828 46.781 31.828 76.594c0 29.859-12.188 56.953-31.828 76.594s-46.781 31.828-76.594 31.828z" fill-rule="evenodd"/>
<path d="m269.53 495.71h409.22c14.625 0 27.938-6 37.641-15.656 9.7031-9.7031 15.656-23.016 15.656-37.641s-6-27.938-15.656-37.641c-9.7031-9.7031-23.016-15.656-37.641-15.656h-409.22c-14.625 0-27.984 6-37.641 15.656-9.7031 9.7031-15.703 23.016-15.703 37.641 0 14.672 6 27.984 15.703 37.641 9.7031 9.7031 23.016 15.656 37.641 15.656zm-257.34-184.74v578.11c0 40.219 16.453 76.734 42.891 103.22 26.438 26.438 63 42.891 103.22 42.891h631.64c40.219 0 76.734-16.406 103.22-42.891 26.438-26.438 42.891-63 42.891-103.22v-40.312l96.141 43.5c17.391 7.875 35.531 11.109 53.438 9.7969 18-1.3125 35.531-7.2188 51.609-17.578 16.078-10.359 28.641-23.906 37.312-39.797 8.6719-15.891 13.266-33.797 13.266-52.828v-383.68c0-19.031-4.5938-36.891-13.266-52.828-8.625-15.891-21.234-29.391-37.312-39.75-16.078-10.359-33.562-16.219-51.609-17.625-17.906-1.3125-36 1.9219-53.438 9.7969l-96.141 43.5v-40.312c0-40.219-16.406-76.734-42.891-103.22-26.438-26.438-63-42.891-103.22-42.891h-631.64c-40.219 0-76.734 16.406-103.22 42.891-26.438 26.438-42.891 63-42.891 103.22zm666.56 239.81h-409.22c-29.859 0-56.953-12.188-76.594-31.828s-31.828-46.734-31.828-76.594 12.188-56.953 31.828-76.594 46.734-31.828 76.594-31.828h409.22c29.859 0 56.953 12.188 76.594 31.828s31.828 46.781 31.828 76.594c0 29.859-12.188 56.953-31.828 76.594s-46.781 31.828-76.594 31.828z" />
</svg>
)
}
Expand Down
25 changes: 25 additions & 0 deletions apps/chat/components/ui/multiplayer-actions.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import type {
ActionMessage,
Attachment,
PlayableAudioStream,
PlayableVideoStream,
} from 'react-agents/types';
import { useLoading } from '@/lib/client/hooks/use-loading';
import * as codecs from '@upstreet/multiplayer/public/audio/ws-codec-runtime-worker.mjs';
Expand Down Expand Up @@ -104,6 +105,10 @@ interface MultiplayerActionsContextType {
waitForFinish: () => Promise<void>
}
removeAudioSource: (stream: PlayableAudioStream) => void
addVideoSource: (stream: PlayableVideoStream) => {
waitForFinish: () => Promise<void>
}
removeVideoSource: (stream: PlayableVideoStream) => void
typingMap: TypingMap
epoch: number
}
Expand Down Expand Up @@ -787,6 +792,22 @@ export function MultiplayerActionsProvider({ children }: MultiplayerActionsProvi
throw new Error('realms not connected');
}
},
addVideoSource: (stream: PlayableVideoStream) => {
if (realms) {
return realms.addVideoSource(stream) as {
waitForFinish: () => Promise<void>
};
} else {
throw new Error('realms not connected');
}
},
removeVideoSource: (stream: PlayableVideoStream) => {
if (realms) {
realms.removeVideoSource(stream);
} else {
throw new Error('realms not connected');
}
},
typingMap,
};
return multiplayerState;
Expand All @@ -809,6 +830,8 @@ export function MultiplayerActionsProvider({ children }: MultiplayerActionsProvi
const agentLeave = multiplayerState.agentLeave;
const addAudioSource = multiplayerState.addAudioSource;
const removeAudioSource = multiplayerState.removeAudioSource;
const addVideoSource = multiplayerState.addVideoSource;
const removeVideoSource = multiplayerState.removeVideoSource;

return (
<MultiplayerActionsContext.Provider
Expand All @@ -830,6 +853,8 @@ export function MultiplayerActionsProvider({ children }: MultiplayerActionsProvi
agentLeave,
addAudioSource,
removeAudioSource,
addVideoSource,
removeVideoSource,
typingMap,
epoch,
}}
Expand Down
Loading

0 comments on commit 0106e2c

Please sign in to comment.