From 17553c0284180ec14ad83318d04aa189ef8ff9cb Mon Sep 17 00:00:00 2001 From: Mathieu Hirel <1053712+ralmn@users.noreply.github.com> Date: Sun, 10 Nov 2024 17:27:40 +0100 Subject: [PATCH] feat: mention mapping --- .dockerignore | 5 ++ .gitignore | 1 + src/configuration/configuration.ts | 23 +++++++ src/constants.ts | 1 + src/helpers/post/make-bluesky-post.ts | 4 +- src/helpers/post/make-mastodon-post.ts | 4 +- src/helpers/post/make-post.ts | 10 ++- .../tweet/__tests__/split-tweet-text.spec.ts | 64 +++++++++++++++---- .../split-tweet-text/split-tweet-text.ts | 49 +++++++++++--- src/index.ts | 2 + src/services/posts-synchronizer.service.ts | 3 + src/types/mentionMapping.ts | 5 ++ 12 files changed, 146 insertions(+), 25 deletions(-) create mode 100644 src/types/mentionMapping.ts diff --git a/.dockerignore b/.dockerignore index 0590d59..5dbe713 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,8 @@ node_modules/ deployment/ docs/ Dockerfile + +# Temporary files +cache*.json +cookies*.json +mention-mapping*.json diff --git a/.gitignore b/.gitignore index 5b4bfdc..cd3cb65 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ coverage/ # Temporary files cache*.json cookies*.json +mention-mapping*.json diff --git a/src/configuration/configuration.ts b/src/configuration/configuration.ts index b14b6f0..cbff301 100644 --- a/src/configuration/configuration.ts +++ b/src/configuration/configuration.ts @@ -4,6 +4,7 @@ import pm2 from "@pm2/io"; import type Counter from "@pm2/io/build/main/utils/metrics/counter"; import type Gauge from "@pm2/io/build/main/utils/metrics/gauge"; import { Scraper } from "@the-convocation/twitter-scraper"; +import * as fs from "fs/promises"; import { createRestAPIClient, mastodon } from "masto"; import ora from "ora"; @@ -13,6 +14,7 @@ import { BLUESKY_PASSWORD, MASTODON_ACCESS_TOKEN, MASTODON_INSTANCE, + MENTION_MAPPING_PATH, SYNC_BLUESKY, SYNC_DRY_RUN, SYNC_MASTODON, @@ -25,6 +27,7 @@ import { getCachedPosts } from "../helpers/cache/get-cached-posts"; import { runMigrations } from "../helpers/cache/run-migrations"; import { TouitomamoutError } from "../helpers/error"; import { oraPrefixer } from "../helpers/logs"; +import { MentionMapping } from "../types/mentionMapping"; import { buildConfigurationRules } from "./build-configuration-rules"; export const configuration = async (): Promise<{ @@ -33,6 +36,7 @@ export const configuration = async (): Promise<{ twitterClient: Scraper; mastodonClient: null | mastodon.rest.Client; blueskyClient: null | AtpAgent; + mentionsMapping: MentionMapping[]; }> => { // Error handling const rules = buildConfigurationRules(); @@ -160,11 +164,30 @@ export const configuration = async (): Promise<{ }); } + let mentionsMapping: MentionMapping[] = []; + //accessSync + try { + const content = (await fs.readFile(MENTION_MAPPING_PATH)).toString(); + mentionsMapping = JSON.parse(content) as MentionMapping[]; + } catch (e) { + const log = ora({ + color: "gray", + prefixText: oraPrefixer("mention-mapping"), + }); + if (e instanceof Error && "code" in e && e.code == "ENOENT") { + log.warn(`No mention mapping file found (${MENTION_MAPPING_PATH})`); + } else { + log.fail(`Error when read ${MENTION_MAPPING_PATH}`); + console.error(e); + } + } + return { mastodonClient, twitterClient, blueskyClient, synchronizedPostsCountAllTime, synchronizedPostsCountThisRun, + mentionsMapping, }; }; diff --git a/src/constants.ts b/src/constants.ts index b25c333..c506d4e 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -44,6 +44,7 @@ export const INSTANCE_ID = (TWITTER_HANDLE ?? "instance") export const STORAGE_DIR = process.env.STORAGE_DIR ?? process.cwd(); export const CACHE_PATH = `${STORAGE_DIR}/cache.${INSTANCE_ID}.json`; export const COOKIES_PATH = `${STORAGE_DIR}/cookies.${INSTANCE_ID}.json`; +export const MENTION_MAPPING_PATH = `${STORAGE_DIR}/mention-mapping.${INSTANCE_ID}.json`; export const SYNC_MASTODON = (process.env.SYNC_MASTODON ?? "false") === "true"; export const SYNC_BLUESKY = (process.env.SYNC_BLUESKY ?? "false") === "true"; export const BACKDATE_BLUESKY_POSTS = diff --git a/src/helpers/post/make-bluesky-post.ts b/src/helpers/post/make-bluesky-post.ts index 5026fb7..28412ed 100644 --- a/src/helpers/post/make-bluesky-post.ts +++ b/src/helpers/post/make-bluesky-post.ts @@ -3,6 +3,7 @@ import { Tweet } from "@the-convocation/twitter-scraper"; import { BLUESKY_IDENTIFIER } from "../../constants"; import { BlueskyCacheChunk, Platform } from "../../types"; +import { MentionMapping } from "../../types/mentionMapping"; import { BlueskyPost } from "../../types/post"; import { getCachedPostChunk } from "../cache/get-cached-post-chunk"; import { splitTextForBluesky } from "../tweet/split-tweet-text"; @@ -10,6 +11,7 @@ import { splitTextForBluesky } from "../tweet/split-tweet-text"; export const makeBlueskyPost = async ( client: AtpAgent, tweet: Tweet, + mentionsMapping: MentionMapping[], ): Promise => { const username = await client .getProfile({ actor: BLUESKY_IDENTIFIER }) @@ -57,7 +59,7 @@ export const makeBlueskyPost = async ( await post.detectFacets(client); // automatically detects mentions and links return { - chunks: await splitTextForBluesky(tweet), + chunks: await splitTextForBluesky(tweet, mentionsMapping), username, replyPost, quotePost, diff --git a/src/helpers/post/make-mastodon-post.ts b/src/helpers/post/make-mastodon-post.ts index 97a87d4..0f62eb3 100644 --- a/src/helpers/post/make-mastodon-post.ts +++ b/src/helpers/post/make-mastodon-post.ts @@ -2,6 +2,7 @@ import { Tweet } from "@the-convocation/twitter-scraper"; import { mastodon } from "masto"; import { Platform } from "../../types"; +import { MentionMapping } from "../../types/mentionMapping"; import { MastodonPost } from "../../types/post"; import { getCachedPosts } from "../cache/get-cached-posts"; import { splitTextForMastodon } from "../tweet/split-tweet-text"; @@ -9,6 +10,7 @@ import { splitTextForMastodon } from "../tweet/split-tweet-text"; export const makeMastodonPost = async ( client: mastodon.rest.Client, tweet: Tweet, + mentionsMapping: MentionMapping[], ): Promise => { const cachedPosts = await getCachedPosts(); @@ -17,7 +19,7 @@ export const makeMastodonPost = async ( .then((account) => account.username); // Get post chunks (including quote in first one when needed) - const chunks = await splitTextForMastodon(tweet, username); + const chunks = await splitTextForMastodon(tweet, mentionsMapping, username); // Get in reply post references let inReplyToId = undefined; diff --git a/src/helpers/post/make-post.ts b/src/helpers/post/make-post.ts index dc0e525..87cea62 100644 --- a/src/helpers/post/make-post.ts +++ b/src/helpers/post/make-post.ts @@ -4,6 +4,7 @@ import { mastodon } from "masto"; import { Ora } from "ora"; import { VOID } from "../../constants"; +import { MentionMapping } from "../../types/mentionMapping"; import { BlueskyPost, MastodonPost, Post } from "../../types/post"; import { oraProgress } from "../logs"; import { getPostExcerpt } from "./get-post-excerpt"; @@ -39,6 +40,7 @@ export const makePost = async ( blueskyClient: AtpAgent | null, log: Ora, counters: { current: number; total: number }, + mentionsMapping: MentionMapping[], ): Promise => { const postExcerpt = getPostExcerpt(tweet.text ?? VOID).padEnd(32, " "); log.color = "magenta"; @@ -52,13 +54,17 @@ export const makePost = async ( // Mastodon post let mastodonPost = null; if (mastodonClient) { - mastodonPost = await makeMastodonPost(mastodonClient, tweet); + mastodonPost = await makeMastodonPost( + mastodonClient, + tweet, + mentionsMapping, + ); } // Bluesky post let blueskyPost = null; if (blueskyClient) { - blueskyPost = await makeBlueskyPost(blueskyClient, tweet); + blueskyPost = await makeBlueskyPost(blueskyClient, tweet, mentionsMapping); } const chunksByPlatform = { diff --git a/src/helpers/tweet/__tests__/split-tweet-text.spec.ts b/src/helpers/tweet/__tests__/split-tweet-text.spec.ts index 3457771..8207f1d 100644 --- a/src/helpers/tweet/__tests__/split-tweet-text.spec.ts +++ b/src/helpers/tweet/__tests__/split-tweet-text.spec.ts @@ -1,6 +1,9 @@ +import { describe } from "node:test"; + import { Tweet } from "@the-convocation/twitter-scraper"; import { MASTODON_INSTANCE } from "../../../constants"; +import { MentionMapping } from "../../../types/mentionMapping"; import { splitTextForBluesky, splitTextForMastodon, @@ -63,9 +66,10 @@ describe("splitTweetText", () => { } as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([ @@ -85,9 +89,10 @@ describe("splitTweetText", () => { } as unknown as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([POST_99_CHARS]); @@ -103,9 +108,10 @@ describe("splitTweetText", () => { } as unknown as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([ @@ -124,9 +130,10 @@ describe("splitTweetText", () => { } as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([POST_299_CHARS]); @@ -142,9 +149,10 @@ describe("splitTweetText", () => { } as unknown as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([ @@ -164,9 +172,10 @@ describe("splitTweetText", () => { } as unknown as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toStrictEqual([ @@ -191,9 +200,10 @@ describe("splitTweetText", () => { } as unknown as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); @@ -223,9 +233,10 @@ describe("splitTweetText", () => { } as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([ @@ -244,9 +255,10 @@ describe("splitTweetText", () => { } as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([ @@ -271,9 +283,10 @@ describe("splitTweetText", () => { } as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([ @@ -294,9 +307,10 @@ describe("splitTweetText", () => { } as Tweet; const mastodonStatuses = await splitTextForMastodon( tweet, + [], MASTODON_USERNAME, ); - const blueskyStatuses = await splitTextForBluesky(tweet); + const blueskyStatuses = await splitTextForBluesky(tweet, []); checkChunksLengthExpectations(mastodonStatuses, blueskyStatuses); expect(mastodonStatuses).toEqual([ @@ -308,4 +322,32 @@ describe("splitTweetText", () => { ]); }); }); + + describe("when mention user with existing mention mapping", () => { + it("should return text with the replaced mention", async () => { + const tweet = { + text: "Hello @ralmn45 how are you ?", + mentions: [{ id: "12345", username: "ralmn45", name: "ralmn" }], + } as Tweet; + const mentionsMapping: MentionMapping[] = [ + { + twitter: "ralmn45", + mastodon: "ralmn@mastodon.xyz", + bluesky: "ralmn.fr", + }, + ]; + + const mastodonStatuses = await splitTextForMastodon( + tweet, + mentionsMapping, + MASTODON_USERNAME, + ); + const blueskyStatuses = await splitTextForBluesky(tweet, mentionsMapping); + + expect(mastodonStatuses).toEqual([ + `Hello @ralmn@mastodon.xyz how are you ?`, + ]); + expect(blueskyStatuses).toEqual([`Hello @ralmn.fr how are you ?`]); + }); + }); }); diff --git a/src/helpers/tweet/split-tweet-text/split-tweet-text.ts b/src/helpers/tweet/split-tweet-text/split-tweet-text.ts index 1169aee..0bc765d 100644 --- a/src/helpers/tweet/split-tweet-text/split-tweet-text.ts +++ b/src/helpers/tweet/split-tweet-text/split-tweet-text.ts @@ -5,13 +5,15 @@ import { MASTODON_MAX_POST_LENGTH, } from "../../../constants"; import { Platform } from "../../../types"; +import { MentionMapping } from "../../../types/mentionMapping"; import { buildChunksFromSplitterEntries } from "./build-chunks-from-splitter-entries"; import { extractWordsAndSpacers } from "./extract-words-and-spacers"; import { getMastodonQuoteLinkSection } from "./get-mastodon-quote-link-section"; const splitTweetText = async ( - { text, quotedStatusId, urls }: Tweet, + { text, quotedStatusId, urls, mentions }: Tweet, platform: Platform, + mentionsMapping: MentionMapping[], mastodonUsername?: string, ): Promise => { const maxChunkSize = @@ -24,19 +26,40 @@ const splitTweetText = async ( mastodonUsername, ); + let tweetText = text!; + + if (mentions != null) { + for (const tweetMention of mentions) { + const mapped = mentionsMapping.find( + (mapping) => tweetMention.username == mapping.twitter, + ); + if (mapped == null) { + continue; + } + const newMention = + platform === Platform.MASTODON ? mapped!.mastodon : mapped!.bluesky; + if (newMention != null) { + tweetText = tweetText.replaceAll( + `@${tweetMention.username}`, + `@${newMention}`, + ); + } + } + } + // Small post optimization if (quotedStatusId) { if (platform === Platform.MASTODON) { // Specific optimization for Mastodon (it has to include a link to the quoted status) - if (text!.length - quotedStatusLinkSection.length <= maxChunkSize) { - return [text + quotedStatusLinkSection]; + if (tweetText.length - quotedStatusLinkSection.length <= maxChunkSize) { + return [tweetText + quotedStatusLinkSection]; } } - } else if (text!.length <= maxChunkSize) { - return [text!]; + } else if (tweetText.length <= maxChunkSize) { + return [tweetText]; } - const entries = extractWordsAndSpacers(text!, urls); + const entries = extractWordsAndSpacers(tweetText, urls); return buildChunksFromSplitterEntries( entries, platform, @@ -46,8 +69,14 @@ const splitTweetText = async ( ); }; -export const splitTextForMastodon = (tweet: Tweet, mastodonUsername: string) => - splitTweetText(tweet, Platform.MASTODON, mastodonUsername); +export const splitTextForMastodon = ( + tweet: Tweet, + mentionsMapping: MentionMapping[], + mastodonUsername: string, +) => + splitTweetText(tweet, Platform.MASTODON, mentionsMapping, mastodonUsername); -export const splitTextForBluesky = (tweet: Tweet) => - splitTweetText(tweet, Platform.BLUESKY); +export const splitTextForBluesky = ( + tweet: Tweet, + mentionsMapping: MentionMapping[], +) => splitTweetText(tweet, Platform.BLUESKY, mentionsMapping); diff --git a/src/index.ts b/src/index.ts index e5f5afd..c03e538 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,6 +18,7 @@ const { synchronizedPostsCountAllTime, synchronizedPostsCountThisRun, blueskyClient, + mentionsMapping, } = await configuration(); /** @@ -47,6 +48,7 @@ const touitomamout = async () => { mastodonClient, blueskyClient, synchronizedPostsCountThisRun, + mentionsMapping, ); synchronizedPostsCountAllTime.set(postsSyncResponse.metrics.totalSynced); diff --git a/src/services/posts-synchronizer.service.ts b/src/services/posts-synchronizer.service.ts index 5c7fea1..8f98f33 100644 --- a/src/services/posts-synchronizer.service.ts +++ b/src/services/posts-synchronizer.service.ts @@ -9,6 +9,7 @@ import { getCachedPosts } from "../helpers/cache/get-cached-posts"; import { oraPrefixer } from "../helpers/logs"; import { makePost } from "../helpers/post/make-post"; import { Media, Metrics, SynchronizerResponse } from "../types"; +import { MentionMapping } from "../types/mentionMapping"; import { blueskySenderService } from "./bluesky-sender.service"; import { mastodonSenderService } from "./mastodon-sender.service"; import { tweetsGetterService } from "./tweets-getter.service"; @@ -21,6 +22,7 @@ export const postsSynchronizerService = async ( mastodonClient: mastodon.rest.Client | null, blueskyClient: AtpAgent | null, synchronizedPostsCountThisRun: Counter.default, + mentionsMapping: MentionMapping[], ): Promise => { const tweets = await tweetsGetterService(twitterClient); @@ -44,6 +46,7 @@ export const postsSynchronizerService = async ( blueskyClient, log, { current: tweetIndex, total: tweets.length }, + mentionsMapping, ); if (!SYNC_DRY_RUN) { diff --git a/src/types/mentionMapping.ts b/src/types/mentionMapping.ts new file mode 100644 index 0000000..4b4b671 --- /dev/null +++ b/src/types/mentionMapping.ts @@ -0,0 +1,5 @@ +export interface MentionMapping { + twitter: string; + mastodon?: string; + bluesky?: string; +}