Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Extra Multimedia Support for Telegram Client #1837

Open
wants to merge 13 commits into
base: develop
Choose a base branch
from
+460 −50
Open

Add Extra Multimedia Support for Telegram Client #1837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
eceb46b
feat: ptb generation action
astinz Dec 22, 2024
5f27d13
minor fix...
astinz Dec 24, 2024
8d92049
Merge branch 'develop' of github.com:elizaOS/eliza into develop
astinz Dec 25, 2024
b48ef3a
Merge branch 'elizaOS:develop' into develop
astinz Dec 26, 2024
13ddd55
Merge branch 'elizaOS:develop' into develop
astinz Dec 26, 2024
2ad7576
Merge branch 'develop' into develop
astinz Dec 29, 2024
ac8bd00
Merge branch 'develop' into develop
astinz Jan 1, 2025
8a5f943
Merge branch 'develop' into develop
astinz Jan 4, 2025
2ade16e
Merge branch 'develop' of github.com:elizaOS/eliza into develop
astinz Jan 4, 2025
b2abea0
Merge branch 'elizaOS:develop' into develop
astinz Jan 4, 2025
2f0683a
Merge remote-tracking branch 'origin/develop' into develop
astinz Jan 4, 2025
faa4d0c
feat: extend telegram client multimedia support
astinz Jan 4, 2025
c80d271
Merge branch 'elizaOS:main' into feature/client-telegram-multimedia
astinz Jan 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 102 additions & 49 deletions packages/client-telegram/src/messageManager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { Message } from "@telegraf/types";
import { Context, Telegraf } from "telegraf";
import { composeContext, elizaLogger, ServiceType, composeRandomUser } from "@elizaos/core";

import { composeContext, elizaLogger, ServiceType } from "@elizaos/core";
import { getEmbeddingZeroVector } from "@elizaos/core";
import {
Content,
Expand All @@ -18,7 +19,7 @@ import { stringToUuid } from "@elizaos/core";
import { generateMessageResponse, generateShouldRespond } from "@elizaos/core";
import { messageCompletionFooter, shouldRespondFooter } from "@elizaos/core";

import { cosineSimilarity, escapeMarkdown } from "./utils";
import { cosineSimilarity } from "./utils";
import {
MESSAGE_CONSTANTS,
TIMING_CONSTANTS,
Expand Down Expand Up @@ -296,8 +297,8 @@ export class MessageManager {
"text" in message
? message.text
: "caption" in message
? (message as any).caption
: "";
? (message as any).caption
: "";

if (!messageText) return false;

Expand Down Expand Up @@ -359,8 +360,8 @@ export class MessageManager {
"text" in message
? message.text
: "caption" in message
? (message as any).caption
: "";
? (message as any).caption
: "";
if (!messageText) return false;

const isReplyToBot =
Expand All @@ -375,7 +376,7 @@ export class MessageManager {
isReplyToBot ||
isMentioned ||
(!this.runtime.character.clientConfig?.telegram
?.shouldRespondOnlyToMentions &&
?.shouldRespondOnlyToMentions &&
hasUsername)
);
}
Expand Down Expand Up @@ -507,8 +508,8 @@ export class MessageManager {
"text" in message
? message.text
: "caption" in message
? (message as any).caption
: "";
? (message as any).caption
: "";

// Check if team member has direct interest first
if (
Expand All @@ -529,8 +530,8 @@ export class MessageManager {
const randomDelay =
Math.floor(
Math.random() *
(TIMING_CONSTANTS.TEAM_MEMBER_DELAY_MAX -
TIMING_CONSTANTS.TEAM_MEMBER_DELAY_MIN)
(TIMING_CONSTANTS.TEAM_MEMBER_DELAY_MAX -
TIMING_CONSTANTS.TEAM_MEMBER_DELAY_MIN)
) + TIMING_CONSTANTS.TEAM_MEMBER_DELAY_MIN; // 1-3 second random delay
await new Promise((resolve) =>
setTimeout(resolve, randomDelay)
Expand All @@ -556,8 +557,8 @@ export class MessageManager {
const leaderResponded = recentMessages.some(
(m) =>
m.userId ===
this.runtime.character.clientConfig?.telegram
?.teamLeaderId &&
this.runtime.character.clientConfig?.telegram
?.teamLeaderId &&
Date.now() - chatState.lastMessageSent < 3000
);

Expand All @@ -578,8 +579,8 @@ export class MessageManager {
const randomDelay =
Math.floor(
Math.random() *
(TIMING_CONSTANTS.LEADER_DELAY_MAX -
TIMING_CONSTANTS.LEADER_DELAY_MIN)
(TIMING_CONSTANTS.LEADER_DELAY_MAX -
TIMING_CONSTANTS.LEADER_DELAY_MIN)
) + TIMING_CONSTANTS.LEADER_DELAY_MIN; // 2-4 second random delay
await new Promise((resolve) =>
setTimeout(resolve, randomDelay)
Expand Down Expand Up @@ -617,7 +618,7 @@ export class MessageManager {
if (chatState?.currentHandler) {
if (
chatState.currentHandler !==
this.bot.botInfo?.id.toString() &&
this.bot.botInfo?.id.toString() &&
this._isTeamMember(chatState.currentHandler)
) {
return false;
Expand All @@ -628,7 +629,7 @@ export class MessageManager {
if (!this._isMessageForMe(message) && this.interestChats[chatId]) {
const recentMessages = this.interestChats[
chatId
].messages.slice(-MESSAGE_CONSTANTS.CHAT_HISTORY_COUNT);
].messages.slice(-MESSAGE_CONSTANTS.CHAT_HISTORY_COUNT);
const ourMessageCount = recentMessages.filter(
(m) => m.userId === this.runtime.agentId
).length;
Expand Down Expand Up @@ -660,7 +661,7 @@ export class MessageManager {
this.runtime.character.templates
?.telegramShouldRespondTemplate ||
this.runtime.character?.templates?.shouldRespondTemplate ||
composeRandomUser(telegramShouldRespondTemplate, 2),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you clarify the reasoning behind removing the composeRandomUser function? I'm curious about the context for this change and whether there's an alternative approach we should be using.

telegramShouldRespondTemplate,
});

const response = await generateShouldRespond({
Expand All @@ -684,15 +685,25 @@ export class MessageManager {
if (content.attachments && content.attachments.length > 0) {
content.attachments.map(async (attachment: Media) => {
if (attachment.contentType.startsWith("image")) {
this.sendImage(ctx, attachment.url, attachment.description);
await this.sendImage(ctx, attachment.url, attachment.description);
} else if (attachment.contentType.startsWith("doc")) {
await this.sendDocument(
ctx,
attachment.url,
attachment.description
);
} else if (attachment.contentType.startsWith("video")) {
await this.sendVideo(ctx, attachment.url, attachment.description);
} else if (attachment.contentType.startsWith("audio")) {
await this.sendAudio(ctx, attachment.url, attachment.description);
}
});
} else {
const chunks = this.splitMessage(content.text);
const sentMessages: Message.TextMessage[] = [];

for (let i = 0; i < chunks.length; i++) {
const chunk = escapeMarkdown(chunks[i]);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you clarify the reasoning behind removing the escapeMarkdown function?

const chunk = chunks[i];
const sentMessage = (await ctx.telegram.sendMessage(
ctx.chat.id,
chunk,
Expand All @@ -712,42 +723,84 @@ export class MessageManager {
}
}

private async sendImage(
private async sendMedia(
ctx: Context,
imagePath: string,
mediaPath: string,
type: "photo" | "video" | "document" | "audio",
caption?: string
): Promise<void> {
try {
if (/^(http|https):\/\//.test(imagePath)) {
const isUrl = /^(http|https):\/\//.test(mediaPath);
const sendFunctionMap = {
photo: ctx.telegram.sendPhoto.bind(ctx.telegram),
video: ctx.telegram.sendVideo.bind(ctx.telegram),
document: ctx.telegram.sendDocument.bind(ctx.telegram),
audio: ctx.telegram.sendAudio.bind(ctx.telegram),
};

if (!sendFunctionMap[type]) {
throw new Error(`Unsupported media type: ${type}`);
}

const sendFunction = sendFunctionMap[type];

if (isUrl) {
// Handle HTTP URLs
await ctx.telegram.sendPhoto(ctx.chat.id, imagePath, {
caption,
});
await sendFunction(ctx.chat.id, mediaPath, { caption });
} else {
// Handle local file paths
if (!fs.existsSync(imagePath)) {
throw new Error(`File not found: ${imagePath}`);
if (!fs.existsSync(mediaPath)) {
throw new Error(`File not found: ${mediaPath}`);
}

const fileStream = fs.createReadStream(imagePath);

await ctx.telegram.sendPhoto(
const fileStream = fs.createReadStream(mediaPath);
await sendFunction(
ctx.chat.id,
{
source: fileStream,
},
{
caption,
}
{ source: fileStream },
{ caption }
);
}

elizaLogger.info(`Image sent successfully: ${imagePath}`);
elizaLogger.info(
`${type.charAt(0).toUpperCase() + type.slice(1)} sent successfully: ${mediaPath}`
);
} catch (error) {
elizaLogger.error("Error sending image:", error);
elizaLogger.error(`Error sending ${type}:`, error);
}
}

private async sendImage(
ctx: Context,
imagePath: string,
caption?: string
): Promise<void> {
await this.sendMedia(ctx, imagePath, "photo", caption);
}

private async sendVideo(
ctx: Context,
videoPath: string,
caption?: string
): Promise<void> {
await this.sendMedia(ctx, videoPath, "video", caption);
}

private async sendDocument(
ctx: Context,
documentPath: string,
caption?: string
): Promise<void> {
await this.sendMedia(ctx, documentPath, "document", caption);
}

private async sendAudio(
ctx: Context,
audioPath: string,
caption?: string
): Promise<void> {
await this.sendMedia(ctx, audioPath, "audio", caption);
}

// Split message into smaller parts
private splitMessage(text: string): string[] {
const chunks: string[] = [];
Expand Down Expand Up @@ -823,8 +876,8 @@ export class MessageManager {
"text" in message
? message.text
: "caption" in message
? (message as any).caption
: "";
? (message as any).caption
: "";

// Add team handling at the start
if (
Expand Down Expand Up @@ -914,7 +967,7 @@ export class MessageManager {
if (
hasInterest ||
this.interestChats[chatId]?.currentHandler ===
this.bot.botInfo?.id.toString()
this.bot.botInfo?.id.toString()
) {
delete this.interestChats[chatId];

Expand Down Expand Up @@ -953,7 +1006,7 @@ export class MessageManager {
) {
this.interestChats[chatId].messages = this.interestChats[
chatId
].messages.slice(-MESSAGE_CONSTANTS.MAX_MESSAGES);
].messages.slice(-MESSAGE_CONSTANTS.MAX_MESSAGES);
}
}
}
Expand Down Expand Up @@ -1018,10 +1071,10 @@ export class MessageManager {
inReplyTo:
"reply_to_message" in message && message.reply_to_message
? stringToUuid(
message.reply_to_message.message_id.toString() +
"-" +
this.runtime.agentId
)
message.reply_to_message.message_id.toString() +
"-" +
this.runtime.agentId
)
: undefined,
};

Expand Down Expand Up @@ -1084,8 +1137,8 @@ export class MessageManager {
const memory: Memory = {
id: stringToUuid(
sentMessage.message_id.toString() +
"-" +
this.runtime.agentId
"-" +
this.runtime.agentId
),
agentId,
userId: agentId,
Expand Down
2 changes: 2 additions & 0 deletions packages/plugin-image-generation/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,3 +354,5 @@ export const imageGenerationPlugin: Plugin = {
evaluators: [],
providers: [],
};

export default imageGenerationPlugin;
Loading
Loading