From 667832a2c0ed3a5becd5fc2e99b9584064e1d544 Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Sun, 23 Jun 2024 20:14:10 -0400 Subject: [PATCH 01/13] adds YouTube CSV parsing --- .../channels/channel.csv | 7 + .../playlists/A playlist-videos.csv | 9 + ... playlist with a duplicate name-videos.csv | 10 + ...ong playlist title 0123456789 ABCD-vid.csv | 7 + ...ong playlist title 0123456789 ABCDE-vi.csv | 7 + ...ong playlist title 0123456789 ABCDEF-v.csv | 7 + ...ong playlist title 0123456789 ABCDEFG-.csv | 7 + ...ong playlist title 0123456789 ABCDEFGH.csv | 7 + ...30\247\331\204\331\206\330\265-videos.csv" | 7 + .../`-=[]_,._~!@#$_^&_()_+{}_-videos.csv | 7 + .../playlists/playlists.csv | 23 + ...0\215\360\237\221\250\360\237\221\251.csv" | 7 + ...0\212\360\237\230\213\360\237\230\214.csv" | 7 + ...0\212\360\237\230\213\360\237\230\214.csv" | 7 + ...0\226\360\237\230\227\360\237\230\230.csv" | 7 + .../video metadata/video recordings.csv | 12 + .../video metadata/videos.csv | 12 + browser/yt/csv.go | 238 +++++++ browser/yt/csv_test.go | 614 ++++++++++++++++++ go.mod | 1 + go.sum | 2 + helpers/fshelper/readcsv.go | 24 + 22 files changed, 1029 insertions(+) create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/channels/channel.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/A playlist-videos.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My playlist with a duplicate name-videos.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCD-vid.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDE-vi.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEF-v.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEFG-.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEFGH.csv create mode 100644 "browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/Z\315\247\314\221\314\223\314\244\315\224a\314\210\314\210\314\207\315\226\314\255l\315\256\314\222\315\253g\314\214\314\232\314\227\315\232o\314\224\315\256\314\207\315\220\314\207\314\231 \330\247\330\256\330\252\330\250\330\247\330\261 \330\247\331\204\331\206\330\265-videos.csv" create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/`-=[]_,._~!@#$_^&_()_+{}_-videos.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/playlists.csv create mode 100644 "browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\221\261\360\237\221\261\360\237\217\273\360\237\221\261\360\237\217\274\360\237\221\261\360\237\217\275\360\237\221\261\360\237\217\276\360\237\221\261\360\237\217\277 \360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\247\237\342\200\215\342\231\202\357\270\217 \360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213\342\200\215\360\237\221\250\360\237\221\251.csv" create mode 100644 "browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200Lorem\360\237\230\201ipsum\360\237\230\202dolor\360\237\230\203sit\360\237\230\204amet\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214.csv" create mode 100644 "browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200orem\360\237\230\201ipsum\360\237\230\202dolor\360\237\230\203sit\360\237\230\204amet\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214.csv" create mode 100644 "browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200\360\237\230\201\360\237\230\202\360\237\230\203\360\237\230\204\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214\360\237\230\215\360\237\230\216\360\237\230\217\360\237\230\220\360\237\230\221\360\237\230\222\360\237\230\223\360\237\230\225\360\237\230\226\360\237\230\227\360\237\230\230.csv" create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/video recordings.csv create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/videos.csv create mode 100644 browser/yt/csv.go create mode 100644 browser/yt/csv_test.go create mode 100644 helpers/fshelper/readcsv.go diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/channels/channel.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/channels/channel.csv new file mode 100644 index 00000000..6b541b40 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/channels/channel.csv @@ -0,0 +1,7 @@ +Channel ID,Channel Title (Original),Channel Visibility +kb3ZF7Rwt2jc2MvVG1kyaze9,Jonathan Stafford,Public + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/A playlist-videos.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/A playlist-videos.csv new file mode 100644 index 00000000..b33f7f73 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/A playlist-videos.csv @@ -0,0 +1,9 @@ +Video ID,Playlist Video Creation Timestamp +a+q6oaXj7dH,2023-12-15T01:48:39+00:00 +pI3tVoMUwz5,2023-12-15T01:48:39+00:00 +PJvZQ6mMSBf,2023-12-15T01:48:39+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My playlist with a duplicate name-videos.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My playlist with a duplicate name-videos.csv new file mode 100644 index 00000000..51790ddf --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My playlist with a duplicate name-videos.csv @@ -0,0 +1,10 @@ +Video ID,Playlist Video Creation Timestamp +d5IMr4n6DIh,2023-12-17T14:16:09+00:00 +PJvZQ6mMSBf,2023-12-17T14:16:09+00:00 +NTOBfooePHb,2023-12-17T14:16:09+00:00 +a+q6oaXj7dH,2023-12-17T14:16:09+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCD-vid.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCD-vid.csv new file mode 100644 index 00000000..868d3419 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCD-vid.csv @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +rl1vcIiguJV,2023-12-18T23:26:44+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDE-vi.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDE-vi.csv new file mode 100644 index 00000000..e3071458 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDE-vi.csv @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +rl1vcIiguJV,2023-12-18T23:26:27+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEF-v.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEF-v.csv new file mode 100644 index 00000000..418c02d4 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEF-v.csv @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +rl1vcIiguJV,2023-12-18T23:26:19+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEFG-.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEFG-.csv new file mode 100644 index 00000000..a2c9ba56 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEFG-.csv @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +rl1vcIiguJV,2023-12-18T23:26:09+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEFGH.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEFGH.csv new file mode 100644 index 00000000..ea8ef01d --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/My very long playlist title 0123456789 ABCDEFGH.csv @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +PJvZQ6mMSBf,2023-12-17T14:37:31+00:00 + + + + + diff --git "a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/Z\315\247\314\221\314\223\314\244\315\224a\314\210\314\210\314\207\315\226\314\255l\315\256\314\222\315\253g\314\214\314\232\314\227\315\232o\314\224\315\256\314\207\315\220\314\207\314\231 \330\247\330\256\330\252\330\250\330\247\330\261 \330\247\331\204\331\206\330\265-videos.csv" "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/Z\315\247\314\221\314\223\314\244\315\224a\314\210\314\210\314\207\315\226\314\255l\315\256\314\222\315\253g\314\214\314\232\314\227\315\232o\314\224\315\256\314\207\315\220\314\207\314\231 \330\247\330\256\330\252\330\250\330\247\330\261 \330\247\331\204\331\206\330\265-videos.csv" new file mode 100644 index 00000000..4702c477 --- /dev/null +++ "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/Z\315\247\314\221\314\223\314\244\315\224a\314\210\314\210\314\207\315\226\314\255l\315\256\314\222\315\253g\314\214\314\232\314\227\315\232o\314\224\315\256\314\207\315\220\314\207\314\231 \330\247\330\256\330\252\330\250\330\247\330\261 \330\247\331\204\331\206\330\265-videos.csv" @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +pI3tVoMUwz5,2023-12-17T14:33:22+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/`-=[]_,._~!@#$_^&_()_+{}_-videos.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/`-=[]_,._~!@#$_^&_()_+{}_-videos.csv new file mode 100644 index 00000000..be338013 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/`-=[]_,._~!@#$_^&_()_+{}_-videos.csv @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +pI3tVoMUwz5,2023-12-17T14:27:45+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/playlists.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/playlists.csv new file mode 100644 index 00000000..2388d68c --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/playlists.csv @@ -0,0 +1,23 @@ +Playlist ID,Add new videos to top,Playlist Description (Original),Playlist Title (Original),Playlist Title (Original) Language,Playlist Create Timestamp,Playlist Update Timestamp,Playlist Video Order,Playlist Visibility +0q2fQnYVgBZ97Pxa0dUTqdHBtk3B/xeyta,False,,😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏,,2023-12-19T22:19:00+00:00,2023-12-19T22:19:09+00:00,Manual,Private ++RSTyiTrFuyMrjlZpuQxw7d3LPzCMc8LaD,False,,😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊tema🙋tis🙌rolod🙍muspi🙎meroL🙏,,2023-12-19T01:07:46+00:00,2023-12-19T01:12:45+00:00,Manual,Private +NdhO/BxkyoiY1OaA98FdsEKotGUIIkenBX,False,,😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏,,2023-12-19T01:06:07+00:00,2023-12-19T01:12:15+00:00,Manual,Private +Vrhwj5jEY4aJ9mssxYIlFS0YEd+YzbSCq3,False,,My very long playlist title 0123456789 ABCD,,2023-12-18T23:26:38+00:00,2023-12-18T23:26:44+00:00,Manual,Private +/dvNHrUBJP17nJrNqt/HaQXHohMf0pR8ZA,False,,My very long playlist title 0123456789 ABCDE,,2023-12-18T23:25:46+00:00,2023-12-18T23:26:27+00:00,Manual,Private +yrM0LUMa/lEHDnJ7UR2cIetuzOcNCWxBnD,False,,My very long playlist title 0123456789 ABCDEF,,2023-12-18T23:25:40+00:00,2023-12-18T23:26:19+00:00,Manual,Private +19XtqwT0XgdNWjT3W9JbQfMEYI8uFiW9yI,False,,My very long playlist title 0123456789 ABCDEFG,,2023-12-18T23:25:31+00:00,2023-12-18T23:26:09+00:00,Manual,Private +8eN2mETnoqHuqDoxXq3nLApeBxdqg7r9qU,False,,My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs,,2023-12-17T14:37:14+00:00,2023-12-17T14:37:31+00:00,Manual,Private +NnnqWkLMzsQ40on1sPO3D5egybOaP2cra/,False,,Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص,,2023-12-17T14:33:16+00:00,2023-12-17T14:33:22+00:00,Manual,Private +CagoNmT9b8xzQ/JDLR1cTyjQ+R2dWIlkho,False,,👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷,,2023-12-17T14:32:59+00:00,2023-12-17T14:33:08+00:00,Manual,Private +89uIs9+aFZ0rj76rdGXO2xeTQh/kz0aMCB,False,,My playlist with a duplicate name,,2023-12-17T14:03:27+00:00,2023-12-17T14:16:09+00:00,Manual,Private +qfyrziKWJZA/u+O7qJ6b4xxx3zjH4zjpED,False,,My playlist with a duplicate name,,2023-12-17T14:03:21+00:00,2023-12-17T14:27:03+00:00,Manual,Private +ozVxmuJGoBR+aT0FBghvOk+/j3a3JmwZPL,False,,My playlist with a duplicate name,,2023-12-17T14:03:17+00:00,2023-12-17T14:27:18+00:00,Manual,Private +yojzPdFgHjBNXsUcmTsmo6g1hTqsIkZUSB,False,,My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs,,2023-12-17T14:02:54+00:00,2023-12-17T14:27:38+00:00,Manual,Private +ykzP/AtWUAgtD6kfpZyk+CCipFNPlh27FA,False,,"`-=[]\;',./~!@#$%^&*()_+{}|:""?",,2023-12-17T13:57:31+00:00,2023-12-17T14:32:42+00:00,Manual,Private +/63ek5JSj2ZcQSXaBiTslzKRSb+kK015UI,False,This is my playlist,A playlist,,2023-12-15T01:47:18+00:00,2023-12-15T01:48:39+00:00,Manual,Private +ldyZEf/SuTx9DgVls+WTopx7BG8Ufi6kl4,False,,Watch later,en_US,2015-03-22T06:43:50+00:00,2024-05-10T01:25:36+00:00,Manual,Private + + + + + diff --git "a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\221\261\360\237\221\261\360\237\217\273\360\237\221\261\360\237\217\274\360\237\221\261\360\237\217\275\360\237\221\261\360\237\217\276\360\237\221\261\360\237\217\277 \360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\247\237\342\200\215\342\231\202\357\270\217 \360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213\342\200\215\360\237\221\250\360\237\221\251.csv" "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\221\261\360\237\221\261\360\237\217\273\360\237\221\261\360\237\217\274\360\237\221\261\360\237\217\275\360\237\221\261\360\237\217\276\360\237\221\261\360\237\217\277 \360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\247\237\342\200\215\342\231\202\357\270\217 \360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213\342\200\215\360\237\221\250\360\237\221\251.csv" new file mode 100644 index 00000000..28d72fbc --- /dev/null +++ "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\221\261\360\237\221\261\360\237\217\273\360\237\221\261\360\237\217\274\360\237\221\261\360\237\217\275\360\237\221\261\360\237\217\276\360\237\221\261\360\237\217\277 \360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\247\237\342\200\215\342\231\202\357\270\217 \360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213\342\200\215\360\237\221\250\360\237\221\251.csv" @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +pI3tVoMUwz5,2023-12-17T14:33:08+00:00 + + + + + diff --git "a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200Lorem\360\237\230\201ipsum\360\237\230\202dolor\360\237\230\203sit\360\237\230\204amet\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214.csv" "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200Lorem\360\237\230\201ipsum\360\237\230\202dolor\360\237\230\203sit\360\237\230\204amet\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214.csv" new file mode 100644 index 00000000..defb1cfe --- /dev/null +++ "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200Lorem\360\237\230\201ipsum\360\237\230\202dolor\360\237\230\203sit\360\237\230\204amet\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214.csv" @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +rl1vcIiguJV,2023-12-19T01:09:33+00:00 + + + + + diff --git "a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200orem\360\237\230\201ipsum\360\237\230\202dolor\360\237\230\203sit\360\237\230\204amet\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214.csv" "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200orem\360\237\230\201ipsum\360\237\230\202dolor\360\237\230\203sit\360\237\230\204amet\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214.csv" new file mode 100644 index 00000000..d4e17b2f --- /dev/null +++ "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200orem\360\237\230\201ipsum\360\237\230\202dolor\360\237\230\203sit\360\237\230\204amet\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214.csv" @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +rl1vcIiguJV,2023-12-19T22:19:09+00:00 + + + + + diff --git "a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200\360\237\230\201\360\237\230\202\360\237\230\203\360\237\230\204\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214\360\237\230\215\360\237\230\216\360\237\230\217\360\237\230\220\360\237\230\221\360\237\230\222\360\237\230\223\360\237\230\225\360\237\230\226\360\237\230\227\360\237\230\230.csv" "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200\360\237\230\201\360\237\230\202\360\237\230\203\360\237\230\204\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214\360\237\230\215\360\237\230\216\360\237\230\217\360\237\230\220\360\237\230\221\360\237\230\222\360\237\230\223\360\237\230\225\360\237\230\226\360\237\230\227\360\237\230\230.csv" new file mode 100644 index 00000000..11b229d9 --- /dev/null +++ "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/\360\237\230\200\360\237\230\201\360\237\230\202\360\237\230\203\360\237\230\204\360\237\230\205\360\237\230\206\360\237\230\207\360\237\230\210\360\237\230\211\360\237\230\212\360\237\230\213\360\237\230\214\360\237\230\215\360\237\230\216\360\237\230\217\360\237\230\220\360\237\230\221\360\237\230\222\360\237\230\223\360\237\230\225\360\237\230\226\360\237\230\227\360\237\230\230.csv" @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +rl1vcIiguJV,2023-12-19T01:09:44+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/video recordings.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/video recordings.csv new file mode 100644 index 00000000..9d333564 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/video recordings.csv @@ -0,0 +1,12 @@ +Video ID,Video Recording Address,Video Recording Altitude,Video Recording Latitude,Video Recording Longitude,Place ID derived from Google's Places API +PJvZQ6mMSBf,,0,0,0, +rl1vcIiguJV,,0,0,0, +pI3tVoMUwz5,The White House,0,38.8977,-77.0365,ChIJ37HL3ry3t4kRv3YLbdhpWXE +d5IMr4n6DIh,,0,0,0, +a+q6oaXj7dH,,0,0,0, +NTOBfooePHb,,0,0,0, + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/videos.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/videos.csv new file mode 100644 index 00000000..2e25b030 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/videos.csv @@ -0,0 +1,12 @@ +Video ID,Approx Duration (ms),Video Audio Language,Video Category,Video Description (Original),Channel ID,Video Title (Original),Privacy,Video State,Video Create Timestamp,Video Publish Timestamp +PJvZQ6mMSBf,55000,,People,A description of Serenade #2,kb3ZF7Rwt2jc2MvVG1kyaze9,Serenade #2,Private,Processed,2016-03-11T11:19:17+00:00, +rl1vcIiguJV,78000,,People,,kb3ZF7Rwt2jc2MvVG1kyaze9,Serenade #1,Private,Processed,2016-03-11T11:20:49+00:00, +pI3tVoMUwz5,16000,,People,,kb3ZF7Rwt2jc2MvVG1kyaze9,I manually set the location,Private,Processed,2023-12-14T00:34:22+00:00,2023-12-14T05:00:21+00:00 +d5IMr4n6DIh,16000,en-US,People,,kb3ZF7Rwt2jc2MvVG1kyaze9,"`-=[]\;',./~!@#$%^&*()_+{}|:""? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص",Private,Processed,2023-12-14T00:36:14+00:00, +a+q6oaXj7dH,16000,en-US,People,A description of a Short video.,kb3ZF7Rwt2jc2MvVG1kyaze9,"`-=[]\;',./~!@#$%^&*()_+{}|:""? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص",Private,Processed,2023-12-14T01:05:57+00:00, +NTOBfooePHb,16000,en-US,People,,kb3ZF7Rwt2jc2MvVG1kyaze9,"`-=[]\;',./~!@#$%^&*()_+{}|:""? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص",Private,Processed,2023-12-17T14:14:46+00:00, + + + + + diff --git a/browser/yt/csv.go b/browser/yt/csv.go new file mode 100644 index 00000000..036d94c1 --- /dev/null +++ b/browser/yt/csv.go @@ -0,0 +1,238 @@ +package yt + +import ( + "golang.org/x/text/unicode/norm" + "strings" + "time" + "unicode/utf16" + + "github.com/simulot/immich-go/helpers/tzone" +) + +// Read from "channels/channel.csv", this file provides metadata about the +// channel +type YouTubeChannel struct { + ChannelID string `csv:"Channel ID"` + Title string `csv:"Channel Title (Original)"` + Visibility string `csv:"Channel Visibility"` +} + +// Read from "playlists/playlists.csv", this file provides a list of all of +// the playlists. Note that this doesn't provide the filename in which the +// playlist's videos can be found, and that is handled by Filename() because +// the process is insane. +type YouTubePlaylist struct { + PlaylistID string `csv:"Playlist ID"` + Description string `csv:"Playlist Description (Original)"` + Title string `csv:"Playlist Title (Original)"` + TitleLanguage string `csv:"Playlist Title (Original) Language"` + CreateTimestamp string `csv:"Playlist Create Timestamp"` + UpdateTimestamp string `csv:"Playlist Update Timestamp"` + VideoOrder string `csv:"Playlist Video Order"` + Visibility string `csv:"Playlist Visibility"` +} + +// Read from "playlists/$playlist-videos.csv", this file provides a list of +// the videos of a particular playlist +type YouTubePlaylistVideo struct { + VideoID string `csv:"Video ID"` + CreateTimestamp string `csv:"Playlist Video Creation Timestamp"` +} + +// Read from "video metadata/video recordings.csv", this file provides +// metadata about the location the video was recorded +type YouTubeVideoRecording struct { + VideoID string `csv:"Video ID"` + Address string `csv:"Video Recording Address"` + Altitude float64 `csv:"Video Recording Altitude"` + Latitude float64 `csv:"Video Recording Latitude"` + Longitude float64 `csv:"Video Recording Longitude"` + PlaceID string `csv:"Place ID derived from Google's Places API"` +} + +// Read from "video metadata/videos.csv", this file provides metadata +// about the video. Note that neither this type nor YouTubeVideoRecording +// provides the filename of the video. There is seemingly no relationship +// between the Duration and the file size, nor between the CreateTimestamp and +// the numbering, so I have chosen to believe that videos are numbered based +// on the order in which they appear in this file because I don't care. +type YouTubeVideo struct { + VideoID string `csv:"Video ID"` + Duration int `csv:"Approx Duration (ms)"` + Language string `csv:"Video Audio Language"` + Category string `csv:"Video Category"` + Description string `csv:"Video Description (Original)"` + ChannelID string `csv:"Channel ID"` + Title string `csv:"Video Title (Original)"` + Privacy string `csv:"Privacy"` + State string `csv:"Video State"` + CreateTimestamp string `csv:"Video Create Timestamp"` + PublishTimestamp string `csv:"Video Publish Timestamp"` +} + +func (ytmd YouTubeChannel) CleanTitle() (string, bool) { + title := strings.Trim(ytmd.Title, " ") + if title != "" { + return title, true + } else { + return "", false + } +} + +func (ytmd YouTubePlaylist) CleanTitle() (string, bool) { + title := strings.Trim(ytmd.Title, " ") + if title != "" { + return title, true + } else { + return "", false + } +} + +// Generates the filename (under playlists) of this playlist +func (ytmd YouTubePlaylist) Filename() string { + // YouTube is no better at creating filenames than is Google Photos. + // + // The process for creating the name of a playlist CSV seems to be: + // 1. Start with the playlist name + // 2. Delete any backslashes, semicolons, asterisks, pipes, + // colons, or question marks + // 3. Replace any apsotrophes, slashes, percents, or quotes with + // underscores + // 4. Append "-videos" to the filename + // 5. Encode as UTF-16 + // 6. If the encoded string is longer than 47 UTF-16 code units, + // truncate to the shortest string that is *at least* 47 code units. + // Functionally this means if the 47th code unit is the middle of a + // Unicode code point, take the remaining code units to create that + // code point. † + // 7. Decode back to a string + // 8. Append ".csv" + // + // If you're thinking murder thoughts, you are correct. + // + // If you're also thinking, "Can't this lead to multiple playlists + // having the same filename?" you are also correct. + // + // † Despite how convoluted as this is, I'm not sure I've gotten the + // algorithm right. I'm sure there are more shennanigans related to + // yet darker corners of Unicode/UTF-8/UTF-16. + + title := ytmd.Title + + title = strings.ReplaceAll(title, "\\", "") + title = strings.ReplaceAll(title, ";", "") + title = strings.ReplaceAll(title, "|", "") + title = strings.ReplaceAll(title, ":", "") + title = strings.ReplaceAll(title, "?", "") + + title = strings.ReplaceAll(title, "'", "_") + title = strings.ReplaceAll(title, "/", "_") + title = strings.ReplaceAll(title, "%", "_") + title = strings.ReplaceAll(title, "*", "_") + title = strings.ReplaceAll(title, "\"", "_") + + title = title + "-videos" + + // I'm not actually sure if this is necessary but I've had enough + // of dealing with this. Hopefully YouTube is already outputting + // normalized Unicode, but there's no reason to have any faith in + // them. + title = norm.NFC.String(title) + + runes := []rune(title) + if len(utf16.Encode(runes)) > 47 { + // Truncate the string until it's <= 47 code units + for len(utf16.Encode(runes)) > 47 { + runes = runes[:len(runes)-1] + } + // If the string is < 47 then add back the last code point + if len(utf16.Encode(runes)) != 47 { + runes = []rune(title)[:len(runes)+1] + } + title = string(runes) + } + + return title + ".csv" +} + +func (ytv YouTubeVideo) Filename() string { + // This is identical to YouTubePlaylist.Filename() except that: + // 1. The length is different!? + // 2. No counter is added by this function + // 3. No file extension is added by this function + // + // All of the proscriptions of YouTubePlaylist.Filename() apply here + // as well + + title := ytv.Title + + title = strings.ReplaceAll(title, "\\", "") + title = strings.ReplaceAll(title, ";", "") + title = strings.ReplaceAll(title, "|", "") + title = strings.ReplaceAll(title, ":", "") + title = strings.ReplaceAll(title, "?", "") + + title = strings.ReplaceAll(title, "'", "_") + title = strings.ReplaceAll(title, "/", "_") + title = strings.ReplaceAll(title, "%", "_") + title = strings.ReplaceAll(title, "*", "_") + title = strings.ReplaceAll(title, "\"", "_") + + // I'm not actually sure if this is necessary but I've had enough + // of dealing with this. Hopefully YouTube is already outputting + // normalized Unicode, but there's no reason to have any faith in + // them. + title = norm.NFC.String(title) + + runes := []rune(title) + if len(utf16.Encode(runes)) > 43 { + // Truncate the string until it's <= 43 code units + for len(utf16.Encode(runes)) > 43 { + runes = runes[:len(runes)-1] + } + // If the string is < 43 then add back the last code point + if len(utf16.Encode(runes)) != 43 { + runes = []rune(title)[:len(runes)+1] + } + title = string(runes) + } + + return title +} +func (ytv YouTubeVideo) CleanTitle() (string, bool) { + title := strings.Trim(ytv.Title, " ") + if title != "" { + return title, true + } else { + return "", false + } +} + +func (ytv YouTubePlaylist) CleanDescription() (string, bool) { + title := strings.Trim(ytv.Description, " ") + if title != "" { + return title, true + } else { + return "", false + } +} + +func (ytv YouTubeVideo) CleanDescription() (string, bool) { + title := strings.Trim(ytv.Description, " ") + if title != "" { + return title, true + } else { + return "", false + } +} + +func (ytv YouTubeVideo) Time() time.Time { + var t time.Time + var err error + t, err = time.Parse(time.RFC3339, ytv.CreateTimestamp) + if err != nil { + + } + local, _ := tzone.Local() + return t.In(local) +} diff --git a/browser/yt/csv_test.go b/browser/yt/csv_test.go new file mode 100644 index 00000000..7b3de935 --- /dev/null +++ b/browser/yt/csv_test.go @@ -0,0 +1,614 @@ +package yt_test + +import ( + "encoding/json" + "io/fs" + "os" + "reflect" + "testing" + "time" + + "github.com/simulot/immich-go/browser/yt" + "github.com/simulot/immich-go/helpers/fshelper" + "github.com/simulot/immich-go/helpers/tzone" +) + +func TestReadYouTubeChannel(t *testing.T) { + want := []yt.YouTubeChannel{ + yt.YouTubeChannel{ + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "Jonathan Stafford", + Visibility: "Public", + }, + } + + var got []yt.YouTubeChannel + fs := os.DirFS("TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/channels/") + got, err := fshelper.ReadCSV[yt.YouTubeChannel](fs, "channel.csv") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if len(got) != len(want) { + t.Errorf("ReadCSV returned %d rows instead of %d", len(got), len(want)) + } + + for i := 0; i < len(want); i++ { + if !reflect.DeepEqual(got[i], want[i]) { + want_json, _ := json.Marshal(want[i]) + got_json, _ := json.Marshal(got[i]) + t.Fatalf("ReadCSV returned\n%s\ninstead of\n%s\nfor index %d", got_json, want_json, i) + } + } +} + +func TestReadYouTubePlaylist(t *testing.T) { + want := []yt.YouTubePlaylist{ + yt.YouTubePlaylist{ + PlaylistID: "0q2fQnYVgBZ97Pxa0dUTqdHBtk3B/xeyta", + Description: "", + Title: "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + TitleLanguage: "", + CreateTimestamp: "2023-12-19T22:19:00+00:00", + UpdateTimestamp: "2023-12-19T22:19:09+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "+RSTyiTrFuyMrjlZpuQxw7d3LPzCMc8LaD", + Description: "", + Title: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊tema🙋tis🙌rolod🙍muspi🙎meroL🙏", + TitleLanguage: "", + CreateTimestamp: "2023-12-19T01:07:46+00:00", + UpdateTimestamp: "2023-12-19T01:12:45+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "NdhO/BxkyoiY1OaA98FdsEKotGUIIkenBX", + Description: "", + Title: "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + TitleLanguage: "", + CreateTimestamp: "2023-12-19T01:06:07+00:00", + UpdateTimestamp: "2023-12-19T01:12:15+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "Vrhwj5jEY4aJ9mssxYIlFS0YEd+YzbSCq3", + Description: "", + Title: "My very long playlist title 0123456789 ABCD", + TitleLanguage: "", + CreateTimestamp: "2023-12-18T23:26:38+00:00", + UpdateTimestamp: "2023-12-18T23:26:44+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "/dvNHrUBJP17nJrNqt/HaQXHohMf0pR8ZA", + Description: "", + Title: "My very long playlist title 0123456789 ABCDE", + TitleLanguage: "", + CreateTimestamp: "2023-12-18T23:25:46+00:00", + UpdateTimestamp: "2023-12-18T23:26:27+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "yrM0LUMa/lEHDnJ7UR2cIetuzOcNCWxBnD", + Description: "", + Title: "My very long playlist title 0123456789 ABCDEF", + TitleLanguage: "", + CreateTimestamp: "2023-12-18T23:25:40+00:00", + UpdateTimestamp: "2023-12-18T23:26:19+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "19XtqwT0XgdNWjT3W9JbQfMEYI8uFiW9yI", + Description: "", + Title: "My very long playlist title 0123456789 ABCDEFG", + TitleLanguage: "", + CreateTimestamp: "2023-12-18T23:25:31+00:00", + UpdateTimestamp: "2023-12-18T23:26:09+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "8eN2mETnoqHuqDoxXq3nLApeBxdqg7r9qU", + Description: "", + Title: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:37:14+00:00", + UpdateTimestamp: "2023-12-17T14:37:31+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "NnnqWkLMzsQ40on1sPO3D5egybOaP2cra/", + Description: "", + Title: "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:33:16+00:00", + UpdateTimestamp: "2023-12-17T14:33:22+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "CagoNmT9b8xzQ/JDLR1cTyjQ+R2dWIlkho", + Description: "", + Title: "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:32:59+00:00", + UpdateTimestamp: "2023-12-17T14:33:08+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "89uIs9+aFZ0rj76rdGXO2xeTQh/kz0aMCB", + Description: "", + Title: "My playlist with a duplicate name", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:03:27+00:00", + UpdateTimestamp: "2023-12-17T14:16:09+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "qfyrziKWJZA/u+O7qJ6b4xxx3zjH4zjpED", + Description: "", + Title: "My playlist with a duplicate name", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:03:21+00:00", + UpdateTimestamp: "2023-12-17T14:27:03+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "ozVxmuJGoBR+aT0FBghvOk+/j3a3JmwZPL", + Description: "", + Title: "My playlist with a duplicate name", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:03:17+00:00", + UpdateTimestamp: "2023-12-17T14:27:18+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "yojzPdFgHjBNXsUcmTsmo6g1hTqsIkZUSB", + Description: "", + Title: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:02:54+00:00", + UpdateTimestamp: "2023-12-17T14:27:38+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "ykzP/AtWUAgtD6kfpZyk+CCipFNPlh27FA", + Description: "", + Title: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"?", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T13:57:31+00:00", + UpdateTimestamp: "2023-12-17T14:32:42+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "/63ek5JSj2ZcQSXaBiTslzKRSb+kK015UI", + Description: "This is my playlist", + Title: "A playlist", + TitleLanguage: "", + CreateTimestamp: "2023-12-15T01:47:18+00:00", + UpdateTimestamp: "2023-12-15T01:48:39+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + yt.YouTubePlaylist{ + PlaylistID: "ldyZEf/SuTx9DgVls+WTopx7BG8Ufi6kl4", + Description: "", + Title: "Watch later", + TitleLanguage: "en_US", + CreateTimestamp: "2015-03-22T06:43:50+00:00", + UpdateTimestamp: "2024-05-10T01:25:36+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + } + + var got []yt.YouTubePlaylist + fs := os.DirFS("TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/") + got, err := fshelper.ReadCSV[yt.YouTubePlaylist](fs, "playlists.csv") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + if len(got) != len(want) { + t.Errorf("ReadCSV returned %d rows instead of %d", len(got), len(want)) + } + + for i := 0; i < len(want); i++ { + if !reflect.DeepEqual(got[i], want[i]) { + got_json, _ := json.Marshal(got[i]) + want_json, _ := json.Marshal(want[i]) + t.Fatalf("ReadCSV returned\n%s\ninstead of\n%s\nfor index %d", got_json, want_json, i) + } + } +} + +func TestReadYouTubePlaylistVideo(t *testing.T) { + want := []yt.YouTubePlaylistVideo{ + yt.YouTubePlaylistVideo{ + VideoID: "a+q6oaXj7dH", + CreateTimestamp: "2023-12-15T01:48:39+00:00", + }, + yt.YouTubePlaylistVideo{ + VideoID: "pI3tVoMUwz5", + CreateTimestamp: "2023-12-15T01:48:39+00:00", + }, + yt.YouTubePlaylistVideo{ + VideoID: "PJvZQ6mMSBf", + CreateTimestamp: "2023-12-15T01:48:39+00:00", + }, + } + + var got []yt.YouTubePlaylistVideo + fs := os.DirFS("TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/") + got, err := fshelper.ReadCSV[yt.YouTubePlaylistVideo](fs, "A playlist-videos.csv") + if err != nil { + t.Errorf("unexpected error: %s", err) + return + } + + if len(got) != len(want) { + t.Errorf("ReadCSV returned %d rows instead of %d", len(got), len(want)) + return + } + + for i := 0; i < len(want); i++ { + if !reflect.DeepEqual(got[i], want[i]) { + got_json, _ := json.Marshal(got[i]) + want_json, _ := json.Marshal(want[i]) + t.Errorf("ReadCSV returned\n%s\ninstead of\n%s\nfor index %d", got_json, want_json, i) + return + } + } +} + +func TestReadYouTubeVideoRecording(t *testing.T) { + want := []yt.YouTubeVideoRecording{ + yt.YouTubeVideoRecording{ + VideoID: "PJvZQ6mMSBf", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + yt.YouTubeVideoRecording{ + VideoID: "rl1vcIiguJV", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + yt.YouTubeVideoRecording{ + VideoID: "pI3tVoMUwz5", + Address: "The White House", + Altitude: 0, + Latitude: 38.8977, + Longitude: -77.0365, + PlaceID: "ChIJ37HL3ry3t4kRv3YLbdhpWXE", + }, + yt.YouTubeVideoRecording{ + VideoID: "d5IMr4n6DIh", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + yt.YouTubeVideoRecording{ + VideoID: "a+q6oaXj7dH", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + yt.YouTubeVideoRecording{ + VideoID: "NTOBfooePHb", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + } + + var got []yt.YouTubeVideoRecording + fs := os.DirFS("TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/") + got, err := fshelper.ReadCSV[yt.YouTubeVideoRecording](fs, "video recordings.csv") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + if len(got) != len(want) { + t.Errorf("ReadCSV returned %d rows instead of %d", len(got), len(want)) + } + + for i := 0; i < len(want); i++ { + if !reflect.DeepEqual(got[i], want[i]) { + got_json, _ := json.Marshal(got[i]) + want_json, _ := json.Marshal(want[i]) + t.Fatalf("ReadCSV returned\n%s\ninstead of\n%s\nfor index %d", got_json, want_json, i) + } + } +} + +func TestReadYouTubeVideo(t *testing.T) { + want := []yt.YouTubeVideo{ + yt.YouTubeVideo{ + VideoID: "PJvZQ6mMSBf", + Duration: 55000, + Language: "", + Category: "People", + Description: "A description of Serenade #2", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "Serenade #2", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2016-03-11T11:19:17+00:00", + PublishTimestamp: "", + }, + yt.YouTubeVideo{ + VideoID: "rl1vcIiguJV", + Duration: 78000, + Language: "", + Category: "People", + Description: "", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "Serenade #1", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2016-03-11T11:20:49+00:00", + PublishTimestamp: "", + }, + yt.YouTubeVideo{ + VideoID: "pI3tVoMUwz5", + Duration: 16000, + Language: "", + Category: "People", + Description: "", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "I manually set the location", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2023-12-14T00:34:22+00:00", + PublishTimestamp: "2023-12-14T05:00:21+00:00", + }, + yt.YouTubeVideo{ + VideoID: "d5IMr4n6DIh", + Duration: 16000, + Language: "en-US", + Category: "People", + Description: "", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2023-12-14T00:36:14+00:00", + PublishTimestamp: "", + }, + yt.YouTubeVideo{ + VideoID: "a+q6oaXj7dH", + Duration: 16000, + Language: "en-US", + Category: "People", + Description: "A description of a Short video.", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2023-12-14T01:05:57+00:00", + PublishTimestamp: "", + }, + yt.YouTubeVideo{ + VideoID: "NTOBfooePHb", + Duration: 16000, + Language: "en-US", + Category: "People", + Description: "", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2023-12-17T14:14:46+00:00", + PublishTimestamp: "", + }, + } + + var got []yt.YouTubeVideo + fs := os.DirFS("TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/video metadata/") + got, err := fshelper.ReadCSV[yt.YouTubeVideo](fs, "videos.csv") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + if len(got) != len(want) { + t.Errorf("ReadCSV returned %d rows instead of %d", len(got), len(want)) + } + + for i := 0; i < len(want); i++ { + if !reflect.DeepEqual(got[i], want[i]) { + got_json, _ := json.Marshal(got[i]) + want_json, _ := json.Marshal(want[i]) + t.Fatalf("ReadCSV returned\n%s\ninstead of\n%s\nfor index %d", got_json, want_json, i) + } + } +} + +func TestCleanChannelTitle(t *testing.T) { + md := yt.YouTubeChannel{ + Title: " ", + } + + title, ok := md.CleanTitle() + if ok { + t.Errorf("CleanTitle() was ok when it should not have been") + } + + md.Title = " ti tle " + title, ok = md.CleanTitle() + if !ok { + t.Errorf("CleanTitle() was not ok when it should have been") + } + if title != "ti tle" { + t.Errorf("CleanTitle() return `%s` when it should have returned `title`", title) + } +} + +func TestCleanPlaylistTitle(t *testing.T) { + md := yt.YouTubePlaylist{ + Title: " ", + } + + title, ok := md.CleanTitle() + if ok { + t.Errorf("CleanTitle() was ok when it should not have been") + } + + md.Title = " ti tle " + title, ok = md.CleanTitle() + if !ok { + t.Errorf("CleanTitle() was not ok when it should have been") + } + if title != "ti tle" { + t.Errorf("CleanTitle() return `%s` when it should have returned `title`", title) + } +} + +func TestPlaylistFilename(t *testing.T) { + dir := os.DirFS("TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists") + playlists, err := fshelper.ReadCSV[yt.YouTubePlaylist](dir, "playlists.csv") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + for i, playlist := range playlists { + if playlist.Title != "Watch later" { + filename := playlist.Filename() + _, err := fs.Stat(dir, filename) + if err != nil { + t.Errorf("couldn't find filename\n%s\nfrom title\n%s\nat index %d:\n%s", filename, playlist.Title, i, err) + } + } + } +} + +func TestVideoFilename(t *testing.T) { + testCases := []struct { + title string + expected string + }{ + { + title: "A description of Serenade #2", + expected: "A description of Serenade #2", + }, + { + title: "Serenade #1", + expected: "Serenade #1", + }, + { + title: "I manually set the location", + expected: "I manually set the location", + }, + { + title: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + expected: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋", + }, + } + + for _, tc := range testCases { + sut := yt.YouTubeVideo{ + Title: tc.title, + } + filename := sut.Filename() + if filename != tc.expected { + t.Errorf("Got\n%s\ninstead of\n%s\nfrom\n%s", filename, tc.expected, tc.title) + } + } +} + +func TestCleanVideoTitle(t *testing.T) { + md := yt.YouTubeVideo{ + Title: " ", + } + + title, ok := md.CleanTitle() + if ok { + t.Errorf("CleanTitle() was ok when it should not have been") + } + + md.Title = " ti tle " + title, ok = md.CleanTitle() + if !ok { + t.Errorf("CleanTitle() was not ok when it should have been") + } + if title != "ti tle" { + t.Errorf("CleanTitle() return `%s` when it should have returned `title`", title) + } +} + +func TestCleanPlaylistDescription(t *testing.T) { + md := yt.YouTubePlaylist{ + Description: " ", + } + + description, ok := md.CleanDescription() + if ok { + t.Errorf("CleanDescription() was ok when it should not have been") + } + + md.Description = " desc rip tion " + description, ok = md.CleanDescription() + if !ok { + t.Errorf("CleanDescription() was not ok when it should have been") + } + if description != "desc rip tion" { + t.Errorf("CleanDescription() return `%s` when it should have returned `desc rip tion`", description) + } +} + +func TestCleanVideoDescription(t *testing.T) { + md := yt.YouTubeVideo{ + Description: " ", + } + + description, ok := md.CleanDescription() + if ok { + t.Errorf("CleanDescription() was ok when it should not have been") + } + + md.Description = " desc rip tion " + description, ok = md.CleanDescription() + if !ok { + t.Errorf("CleanDescription() was not ok when it should have been") + } + if description != "desc rip tion" { + t.Errorf("CleanDescription() return `%s` when it should have returned `desc rip tion`", description) + } +} + +func TestYouTubeVideoTime(t *testing.T) { + local, _ := tzone.Local() + want := time.Date(int(2023), time.December, int(14), int(1), int(5), int(57), int(0), time.UTC).In(local) + + md := yt.YouTubeVideo{ + CreateTimestamp: "2023-12-14T01:05:57+00:00", + } + got := md.Time() + if got != want { + t.Fatalf("YouTubeVideo.Time() returned %s instead of %s", got, want) + } +} diff --git a/go.mod b/go.mod index 7fbf35e0..270e8884 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/gdamore/tcell/v2 v2.7.4 + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.5.1 github.com/kr/pretty v0.3.1 diff --git a/go.sum b/go.sum index 2db40562..b188040d 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= diff --git a/helpers/fshelper/readcsv.go b/helpers/fshelper/readcsv.go new file mode 100644 index 00000000..798835df --- /dev/null +++ b/helpers/fshelper/readcsv.go @@ -0,0 +1,24 @@ +package fshelper + +import ( + "github.com/gocarina/gocsv" + "io/fs" +) + +// ReadCSV reads a CSV file from the provided file system (fs.FS) +// with the given name and unmarshals it into the provided type T. + +func ReadCSV[T any](FSys fs.FS, name string) ([]T, error) { + var objects []T + b, err := fs.ReadFile(FSys, name) + if err != nil { + return nil, err + } + + err = gocsv.UnmarshalBytes(b, &objects) + if err != nil { + return nil, err + } + + return objects, nil +} From 995e9161df473f2b5a76a986923cbb9370f5d09a Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Fri, 28 Jun 2024 20:37:03 -0400 Subject: [PATCH 02/13] implements Prepare/Browse --- .../videos/I manually set the location.mp4 | 1 + .../videos/Serenade #1.mp4 | 1 + .../videos/Serenade #2.mp4 | 1 + ...70\217\342\200\215\360\237\222\213(1).mp4" | 1 + ...70\217\342\200\215\360\237\222\213(2).mp4" | 1 + ...7\270\217\342\200\215\360\237\222\213.mp4" | 1 + browser/yt/youtube.go | 204 +++++ browser/yt/youtube_test.go | 746 ++++++++++++++++++ 8 files changed, 956 insertions(+) create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/I manually set the location.mp4 create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/Serenade #1.mp4 create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/Serenade #2.mp4 create mode 100644 "browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213(1).mp4" create mode 100644 "browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213(2).mp4" create mode 100644 "browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213.mp4" create mode 100644 browser/yt/youtube.go create mode 100644 browser/yt/youtube_test.go diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/I manually set the location.mp4 b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/I manually set the location.mp4 new file mode 100644 index 00000000..81c545ef --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/I manually set the location.mp4 @@ -0,0 +1 @@ +1234 diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/Serenade #1.mp4 b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/Serenade #1.mp4 new file mode 100644 index 00000000..e56e15bb --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/Serenade #1.mp4 @@ -0,0 +1 @@ +12345 diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/Serenade #2.mp4 b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/Serenade #2.mp4 new file mode 100644 index 00000000..9f358a4a --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/Serenade #2.mp4 @@ -0,0 +1 @@ +123456 diff --git "a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213(1).mp4" "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213(1).mp4" new file mode 100644 index 00000000..48082f72 --- /dev/null +++ "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213(1).mp4" @@ -0,0 +1 @@ +12 diff --git "a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213(2).mp4" "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213(2).mp4" new file mode 100644 index 00000000..190a1803 --- /dev/null +++ "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213(2).mp4" @@ -0,0 +1 @@ +123 diff --git "a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213.mp4" "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213.mp4" new file mode 100644 index 00000000..d00491fd --- /dev/null +++ "b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/videos/`-=[]_,._~!@#$_^&_()_+{}_ \360\237\221\261\360\237\217\273\360\237\247\237\342\200\215\342\231\200\357\270\217\360\237\221\250\342\200\215\342\235\244\357\270\217\342\200\215\360\237\222\213.mp4" @@ -0,0 +1 @@ +1 diff --git a/browser/yt/youtube.go b/browser/yt/youtube.go new file mode 100644 index 00000000..cc7ab5b4 --- /dev/null +++ b/browser/yt/youtube.go @@ -0,0 +1,204 @@ +package yt + +import ( + "context" + "io/fs" + "strconv" + + "github.com/simulot/immich-go/browser" + "github.com/simulot/immich-go/helpers/fileevent" + "github.com/simulot/immich-go/helpers/fshelper" + "github.com/simulot/immich-go/immich" +) + +type SynthesizedYouTubeVideo struct { + Channel *YouTubeChannel + Playlists []*YouTubePlaylist + Video *YouTubeVideo + Recording *YouTubeVideoRecording + Fsys fs.FS + Filename string +} + +type Takeout struct { + fsyss []fs.FS + videos []*SynthesizedYouTubeVideo + log *fileevent.Recorder + sm immich.SupportedMedia +} + +func NewTakeout(ctx context.Context, l *fileevent.Recorder, sm immich.SupportedMedia, fsyss ...fs.FS) (*Takeout, error) { + to := Takeout{ + fsyss: fsyss, + videos: []*SynthesizedYouTubeVideo{}, + log: l, + sm: sm, + } + + return &to, nil +} + + +// Prepare scans all files to build gather and aggregate the metadata +func (to *Takeout) Prepare(ctx context.Context) error { + for _, fsys := range to.fsyss { + tofs, err := fs.Sub(fsys, "Takeout") + if err != nil { + return err + } + ytfs, err := fs.Sub(tofs, "YouTube and YouTube Music") + if err != nil { + return err + } + cfs, err := fs.Sub(ytfs, "channels") + if err != nil { + return err + } + pfs, err := fs.Sub(ytfs, "playlists") + if err != nil { + return err + } + vfs, err := fs.Sub(ytfs, "videos") + if err != nil { + return err + } + vmfs, err := fs.Sub(ytfs, "video metadata") + if err != nil { + return err + } + + // ChannelID => YouTubeChannel + channels := map[string]*YouTubeChannel{} + ytchannels, err := fshelper.ReadCSV[YouTubeChannel](cfs, "channel.csv") + if err != nil { + return err + } + for i, _ := range ytchannels { + channel := ytchannels[i] + channels[channel.ChannelID] = &channel + } + + // VideoID => YouTubePlaylist + playlists := map[string][]*YouTubePlaylist{} + ytplaylists, err := fshelper.ReadCSV[YouTubePlaylist](pfs, "playlists.csv") + if err != nil { + return err + } + for i, _ := range ytplaylists { + playlist := ytplaylists[i] + if playlist.Title == "Watch later" { + continue + } + + ytplaylistvideos, err := fshelper.ReadCSV[YouTubePlaylistVideo](pfs, playlist.Filename()) + if err != nil { + return err + } + for j, _ := range ytplaylistvideos { + playlistvideo := ytplaylistvideos[j] + playlists[playlistvideo.VideoID] = append(playlists[playlistvideo.VideoID], &playlist) + } + } + + // VideoID => YouTubeVideoRecording + recordings := map[string]*YouTubeVideoRecording{} + ytrecordings, err := fshelper.ReadCSV[YouTubeVideoRecording](vmfs, "video recordings.csv") + if err != nil { + return err + } + for i, _ := range ytrecordings { + recording := ytrecordings[i] + recordings[recording.VideoID] = &recording + } + + // Finally, add the other metadata to the videos + videos, err := fshelper.ReadCSV[YouTubeVideo](vmfs, "videos.csv") + if err != nil { + return err + } + + filenames := map[string]int{} + for i, _ := range videos { + video := videos[i] + + filename := video.Filename() + count, ok := filenames[filename] + if !ok { + filenames[filename] = 1 + } else { + filenames[filename] = count + 1 + filename = filename + "(" + strconv.Itoa(count) + ")" + } + filename += ".mp4" + + synth := SynthesizedYouTubeVideo{ + Channel: channels[video.ChannelID], + Playlists: playlists[video.VideoID], + Video: &video, + Recording: recordings[video.VideoID], + Fsys: vfs, + Filename: filename, + } + to.videos = append(to.videos, &synth) + } + } + + return nil +} + +// Browse returns a channel of assets +func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { + assetChan := make(chan *browser.LocalAssetFile) + + go func() { + defer close(assetChan) + for _, video := range to.videos { + fileinfo, err := fs.Stat(video.Fsys, video.Filename) + if err != nil { + continue + } + + albums := []browser.LocalAlbum{} + for _, playlist := range video.Playlists { + album := browser.LocalAlbum{ + Path: playlist.Title, + Name: playlist.Title, + } + albums = append(albums, album) + } + + a := browser.LocalAssetFile{ + FileName: video.Filename, + Title: video.Video.Title, + Description: video.Video.Description, + Albums: albums, + + DateTaken: video.Video.Time(), + Latitude: video.Recording.Latitude, + Longitude: video.Recording.Longitude, + Altitude: video.Recording.Altitude, + + Trashed: false, + Archived: false, + FromPartner: false, + Favorite: false, + + FSys: video.Fsys, + FileSize: int(fileinfo.Size()), + } + + select{ + case <-ctx.Done(): + assetChan <- &browser.LocalAssetFile{Err: ctx.Err()} + case assetChan <- &a: + } + } + }() + return assetChan +} + +// Only exists for testing + +func (to *Takeout) Videos() []*SynthesizedYouTubeVideo { + return to.videos +} diff --git a/browser/yt/youtube_test.go b/browser/yt/youtube_test.go new file mode 100644 index 00000000..90a564c0 --- /dev/null +++ b/browser/yt/youtube_test.go @@ -0,0 +1,746 @@ +package yt_test + +import ( + "context" + "encoding/json" + "io" + "io/fs" + "log/slog" + "os" + "reflect" + "slices" + "sort" + "testing" + "time" + + "github.com/simulot/immich-go/browser" + "github.com/simulot/immich-go/browser/yt" + "github.com/simulot/immich-go/helpers/fileevent" + "github.com/simulot/immich-go/helpers/tzone" + "github.com/simulot/immich-go/immich" +) + +type SynthesizedYouTubeVideosByPlaylistID []*yt.YouTubePlaylist +func (a SynthesizedYouTubeVideosByPlaylistID) Len() int { return len(a) } +func (a SynthesizedYouTubeVideosByPlaylistID) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a SynthesizedYouTubeVideosByPlaylistID) Less(i, j int) bool { return a[i].PlaylistID < a[j].PlaylistID } + +type LocalAlbumsByName []browser.LocalAlbum +func (a LocalAlbumsByName ) Len() int { return len(a) } +func (a LocalAlbumsByName ) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a LocalAlbumsByName ) Less(i, j int) bool { return a[i].Name < a[j].Name } + +func TestPrepareAndBrowse(t *testing.T) { + channel := yt.YouTubeChannel { + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "Jonathan Stafford", + Visibility: "Public", + } + + // YouTubePlaylist.Filename() => []YouTubePlaylist + playlists := map[string][]*yt.YouTubePlaylist{ + "A playlist-videos.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "/63ek5JSj2ZcQSXaBiTslzKRSb+kK015UI", + Description: "This is my playlist", + Title: "A playlist", + TitleLanguage: "", + CreateTimestamp: "2023-12-15T01:47:18+00:00", + UpdateTimestamp: "2023-12-15T01:48:39+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "My playlist with a duplicate name-videos.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "89uIs9+aFZ0rj76rdGXO2xeTQh/kz0aMCB", + Description: "", + Title: "My playlist with a duplicate name", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:03:27+00:00", + UpdateTimestamp: "2023-12-17T14:16:09+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + &yt.YouTubePlaylist{ + PlaylistID: "qfyrziKWJZA/u+O7qJ6b4xxx3zjH4zjpED", + Description: "", + Title: "My playlist with a duplicate name", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:03:21+00:00", + UpdateTimestamp: "2023-12-17T14:27:03+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + &yt.YouTubePlaylist{ + PlaylistID: "ozVxmuJGoBR+aT0FBghvOk+/j3a3JmwZPL", + Description: "", + Title: "My playlist with a duplicate name", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:03:17+00:00", + UpdateTimestamp: "2023-12-17T14:27:18+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "My very long playlist title 0123456789 ABCD-vid.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "Vrhwj5jEY4aJ9mssxYIlFS0YEd+YzbSCq3", + Description: "", + Title: "My very long playlist title 0123456789 ABCD", + TitleLanguage: "", + CreateTimestamp: "2023-12-18T23:26:38+00:00", + UpdateTimestamp: "2023-12-18T23:26:44+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "My very long playlist title 0123456789 ABCDE-vi.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "/dvNHrUBJP17nJrNqt/HaQXHohMf0pR8ZA", + Description: "", + Title: "My very long playlist title 0123456789 ABCDE", + TitleLanguage: "", + CreateTimestamp: "2023-12-18T23:25:46+00:00", + UpdateTimestamp: "2023-12-18T23:26:27+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "My very long playlist title 0123456789 ABCDEF-v.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "yrM0LUMa/lEHDnJ7UR2cIetuzOcNCWxBnD", + Description: "", + Title: "My very long playlist title 0123456789 ABCDEF", + TitleLanguage: "", + CreateTimestamp: "2023-12-18T23:25:40+00:00", + UpdateTimestamp: "2023-12-18T23:26:19+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "My very long playlist title 0123456789 ABCDEFG-.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "19XtqwT0XgdNWjT3W9JbQfMEYI8uFiW9yI", + Description: "", + Title: "My very long playlist title 0123456789 ABCDEFG", + TitleLanguage: "", + CreateTimestamp: "2023-12-18T23:25:31+00:00", + UpdateTimestamp: "2023-12-18T23:26:09+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "My very long playlist title 0123456789 ABCDEFGH.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "8eN2mETnoqHuqDoxXq3nLApeBxdqg7r9qU", + Description: "", + Title: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:37:14+00:00", + UpdateTimestamp: "2023-12-17T14:37:31+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + &yt.YouTubePlaylist{ + PlaylistID: "yojzPdFgHjBNXsUcmTsmo6g1hTqsIkZUSB", + Description: "", + Title: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:02:54+00:00", + UpdateTimestamp: "2023-12-17T14:27:38+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص-videos.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "NnnqWkLMzsQ40on1sPO3D5egybOaP2cra/", + Description: "", + Title: "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:33:16+00:00", + UpdateTimestamp: "2023-12-17T14:33:22+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "`-=[]_,._~!@#$_^&_()_+{}_-videos.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "ykzP/AtWUAgtD6kfpZyk+CCipFNPlh27FA", + Description: "", + Title: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"?", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T13:57:31+00:00", + UpdateTimestamp: "2023-12-17T14:32:42+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "CagoNmT9b8xzQ/JDLR1cTyjQ+R2dWIlkho", + Description: "", + Title: "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷", + TitleLanguage: "", + CreateTimestamp: "2023-12-17T14:32:59+00:00", + UpdateTimestamp: "2023-12-17T14:33:08+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "NdhO/BxkyoiY1OaA98FdsEKotGUIIkenBX", + Description: "", + Title: "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + TitleLanguage: "", + CreateTimestamp: "2023-12-19T01:06:07+00:00", + UpdateTimestamp: "2023-12-19T01:12:15+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "0q2fQnYVgBZ97Pxa0dUTqdHBtk3B/xeyta", + Description: "", + Title: "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + TitleLanguage: "", + CreateTimestamp: "2023-12-19T22:19:00+00:00", + UpdateTimestamp: "2023-12-19T22:19:09+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘.csv": { + &yt.YouTubePlaylist{ + PlaylistID: "+RSTyiTrFuyMrjlZpuQxw7d3LPzCMc8LaD", + Description: "", + Title: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊tema🙋tis🙌rolod🙍muspi🙎meroL🙏", + TitleLanguage: "", + CreateTimestamp: "2023-12-19T01:07:46+00:00", + UpdateTimestamp: "2023-12-19T01:12:45+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, + }, + } + + fsys := os.DirFS("TEST_DATA/20240623T224719") + takeout, err := fs.Sub(fsys, "Takeout") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + youtube, err := fs.Sub(takeout, "YouTube and YouTube Music") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + videos, err := fs.Sub(youtube, "videos") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + wantVideos := []*yt.SynthesizedYouTubeVideo{ + &yt.SynthesizedYouTubeVideo{ + Channel: &channel, + Playlists: slices.Concat(playlists["A playlist-videos.csv"], + playlists["My playlist with a duplicate name-videos.csv"], + playlists["My very long playlist title 0123456789 ABCDEFGH.csv"]), + Video: &yt.YouTubeVideo{ + VideoID: "PJvZQ6mMSBf", + Duration: 55000, + Language: "", + Category: "People", + Description: "A description of Serenade #2", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "Serenade #2", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2016-03-11T11:19:17+00:00", + PublishTimestamp: "", + }, + Recording: &yt.YouTubeVideoRecording{ + VideoID: "PJvZQ6mMSBf", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + Fsys: videos, + Filename: "Serenade #2.mp4", + }, + &yt.SynthesizedYouTubeVideo{ + Channel: &channel, + Playlists: slices.Concat(playlists["😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘.csv"], + playlists["😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌.csv"], + playlists["My very long playlist title 0123456789 ABCDEFG-.csv"], + playlists["My very long playlist title 0123456789 ABCDEF-v.csv"], + playlists["My very long playlist title 0123456789 ABCDE-vi.csv"], + playlists["My very long playlist title 0123456789 ABCD-vid.csv"], + playlists["😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌.csv"]), + Video: &yt.YouTubeVideo{ + VideoID: "rl1vcIiguJV", + Duration: 78000, + Language: "", + Category: "People", + Description: "", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "Serenade #1", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2016-03-11T11:20:49+00:00", + PublishTimestamp: "", + }, + Recording: &yt.YouTubeVideoRecording{ + VideoID: "rl1vcIiguJV", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + Fsys: videos, + Filename: "Serenade #1.mp4", + }, + &yt.SynthesizedYouTubeVideo{ + Channel: &channel, + Playlists: slices.Concat(playlists["`-=[]_,._~!@#$_^&_()_+{}_-videos.csv"], + playlists["A playlist-videos.csv"], + playlists["👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩.csv"], + playlists["Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص-videos.csv"]), + Video: &yt.YouTubeVideo{ + VideoID: "pI3tVoMUwz5", + Duration: 16000, + Language: "", + Category: "People", + Description: "", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "I manually set the location", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2023-12-14T00:34:22+00:00", + PublishTimestamp: "2023-12-14T05:00:21+00:00", + }, + Recording: &yt.YouTubeVideoRecording{ + VideoID: "pI3tVoMUwz5", + Address: "The White House", + Altitude: 0, + Latitude: 38.8977, + Longitude: -77.0365, + PlaceID: "ChIJ37HL3ry3t4kRv3YLbdhpWXE", + }, + Fsys: videos, + Filename: "I manually set the location.mp4", + }, + &yt.SynthesizedYouTubeVideo{ + Channel: &channel, + Playlists: slices.Concat(playlists["My playlist with a duplicate name-videos.csv"]), + Video: &yt.YouTubeVideo{ + VideoID: "d5IMr4n6DIh", + Duration: 16000, + Language: "en-US", + Category: "People", + Description: "", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2023-12-14T00:36:14+00:00", + PublishTimestamp: "", + }, + Recording: &yt.YouTubeVideoRecording{ + VideoID: "d5IMr4n6DIh", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + Fsys: videos, + Filename: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋.mp4", + }, + &yt.SynthesizedYouTubeVideo{ + Channel: &channel, + Playlists: slices.Concat(playlists["A playlist-videos.csv"], + playlists["My playlist with a duplicate name-videos.csv"]), + Video: &yt.YouTubeVideo{ + VideoID: "a+q6oaXj7dH", + Duration: 16000, + Language: "en-US", + Category: "People", + Description: "A description of a Short video.", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2023-12-14T01:05:57+00:00", + PublishTimestamp: "", + }, + Recording: &yt.YouTubeVideoRecording{ + VideoID: "a+q6oaXj7dH", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + Fsys: videos, + Filename: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋(1).mp4", + }, + &yt.SynthesizedYouTubeVideo{ + Channel: &channel, + Playlists: slices.Concat(playlists["My playlist with a duplicate name-videos.csv"]), + Video: &yt.YouTubeVideo{ + VideoID: "NTOBfooePHb", + Duration: 16000, + Language: "en-US", + Category: "People", + Description: "", + ChannelID: "kb3ZF7Rwt2jc2MvVG1kyaze9", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Privacy: "Private", + State: "Processed", + CreateTimestamp: "2023-12-17T14:14:46+00:00", + PublishTimestamp: "", + }, + Recording: &yt.YouTubeVideoRecording{ + VideoID: "NTOBfooePHb", + Address: "", + Altitude: 0, + Latitude: 0, + Longitude: 0, + PlaceID: "", + }, + Fsys: videos, + Filename: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋(2).mp4", + }, + } + + ctx := context.Background() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + l := fileevent.NewRecorder(log, false) + sm := immich.DefaultSupportedMedia + + to, err := yt.NewTakeout(ctx, l, sm, fsys) + if err != nil{ + t.Fatalf("unexpected error: %s", err) + } + + err = to.Prepare(ctx) + if err != nil{ + t.Fatalf("unexpected error: %s", err) + } + + // Test Prepare + gotVideos := to.Videos() + if len(gotVideos) != len(wantVideos) { + t.Errorf("Prepare returned %d videos instead of %d", len(gotVideos), len(wantVideos)) + } + + for i, _ := range gotVideos { + // The order of the playlists in the data we read depends on + // the order of the playlists in playlists.csv, which seems to + // be random. Also we don't really care in the first place, + // so just make it predictable for the test: + sort.Sort(SynthesizedYouTubeVideosByPlaylistID(gotVideos[i].Playlists)) + sort.Sort(SynthesizedYouTubeVideosByPlaylistID(wantVideos[i].Playlists)) + + if !reflect.DeepEqual(gotVideos[i], wantVideos[i]) { + want_json, _ := json.MarshalIndent(wantVideos[i], "", " ") + got_json, _ := json.MarshalIndent(gotVideos[i], "", " ") + t.Fatalf("Prepare returned\n%s\ninstead of\n%s\nfor index %d", got_json, want_json, i) + } + } + + local, _ := tzone.Local() + + wantLafs := []*browser.LocalAssetFile{ + &browser.LocalAssetFile{ + FileName: "Serenade #2.mp4", + Title: "Serenade #2", + Description: "A description of Serenade #2", + Albums: []browser.LocalAlbum{ + browser.LocalAlbum{ + //Path: "A playlist-videos.csv", + Path: "A playlist", + Name: "A playlist", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv", + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv", + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv", + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My very long playlist title 0123456789 ABCDEFGH.csv", + Path: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + Name: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + }, + browser.LocalAlbum{ + //Path: "My very long playlist title 0123456789 ABCDEFGH.csv", + Path: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + Name: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + }, + }, + + DateTaken: time.Date(int(2016), time.March, int(11), int(11), int(19), int(17), int(0), time.UTC).In(local), + Latitude: 0, + Longitude: 0, + Altitude: 0, + + Trashed: false, + Archived: false, + FromPartner: false, + Favorite: false, + + FSys: videos, + FileSize: 7, + }, + &browser.LocalAssetFile{ + FileName: "Serenade #1.mp4", + Title: "Serenade #1", + Description: "", + Albums: []browser.LocalAlbum{ + browser.LocalAlbum{ + //Path: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘.csv" + Path: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊tema🙋tis🙌rolod🙍muspi🙎meroL🙏", + Name: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊tema🙋tis🙌rolod🙍muspi🙎meroL🙏", + }, + browser.LocalAlbum{ + //Path: "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌.csv" + Path: "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + Name: "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + }, + browser.LocalAlbum{ + //Path: "My very long playlist title 0123456789 ABCDEFG-.csv" + Path: "My very long playlist title 0123456789 ABCDEFG", + Name: "My very long playlist title 0123456789 ABCDEFG", + }, + browser.LocalAlbum{ + //Path: "My very long playlist title 0123456789 ABCDEF-v.csv" + Path: "My very long playlist title 0123456789 ABCDEF", + Name: "My very long playlist title 0123456789 ABCDEF", + }, + browser.LocalAlbum{ + //Path: "My very long playlist title 0123456789 ABCDE-vi.csv" + Path: "My very long playlist title 0123456789 ABCDE", + Name: "My very long playlist title 0123456789 ABCDE", + }, + browser.LocalAlbum{ + //Path: "My very long playlist title 0123456789 ABCD-vid.csv" + Path: "My very long playlist title 0123456789 ABCD", + Name: "My very long playlist title 0123456789 ABCD", + }, + browser.LocalAlbum{ + //Path: "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌.csv" + Path: "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + Name: "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + }, + }, + + DateTaken: time.Date(int(2016), time.March, int(11), int(11), int(20), int(49), int(0), time.UTC).In(local), + Latitude: 0, + Longitude: 0, + Altitude: 0, + + Trashed: false, + Archived: false, + FromPartner: false, + Favorite: false, + + FSys: videos, + FileSize: 6, + }, + &browser.LocalAssetFile{ + FileName: "I manually set the location.mp4", + Title: "I manually set the location", + Description: "", + Albums: []browser.LocalAlbum{ + browser.LocalAlbum{ + //Path: "`-=[]_,._~!@#$_^&_()_+{}_-videos.csv" + Path: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"?", + Name: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"?", + }, + browser.LocalAlbum{ + //Path: "A playlist-videos.csv" + Path: "A playlist", + Name: "A playlist", + }, + browser.LocalAlbum{ + //Path: "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩.csv" + Path: "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷", + Name: "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷", + }, + browser.LocalAlbum{ + //Path: "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص-videos.csv" + Path: "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Name: "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + }, + }, + + DateTaken: time.Date(int(2023), time.December, int(14), int(0), int(34), int(22), int(0), time.UTC).In(local), + Latitude: 38.8977, + Longitude: -77.0365, + Altitude: 0, + + Trashed: false, + Archived: false, + FromPartner: false, + Favorite: false, + + FSys: videos, + FileSize: 5, + }, + &browser.LocalAssetFile{ + FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋.mp4", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Description: "", + Albums: []browser.LocalAlbum{ + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + }, + + DateTaken: time.Date(int(2023), time.December, int(14), int(0), int(36), int(14), int(0), time.UTC).In(local), + Latitude: 0, + Longitude: 0, + Altitude: 0, + + Trashed: false, + Archived: false, + FromPartner: false, + Favorite: false, + + FSys: videos, + FileSize: 2, + }, + &browser.LocalAssetFile{ + FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋(1).mp4", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Description: "A description of a Short video.", + Albums: []browser.LocalAlbum{ + browser.LocalAlbum{ + //Path: "A playlist-videos.csv" + Path: "A playlist", + Name: "A playlist", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + }, + + DateTaken: time.Date(int(2023), time.December, int(14), int(1), int(5), int(57), int(0), time.UTC).In(local), + Latitude: 0, + Longitude: 0, + Altitude: 0, + + Trashed: false, + Archived: false, + FromPartner: false, + Favorite: false, + + FSys: videos, + FileSize: 3, + }, + &browser.LocalAssetFile{ + FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋(2).mp4", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Description: "", + Albums: []browser.LocalAlbum{ + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + browser.LocalAlbum{ + //Path: "My playlist with a duplicate name-videos.csv + Path: "My playlist with a duplicate name", + Name: "My playlist with a duplicate name", + }, + }, + + DateTaken: time.Date(int(2023), time.December, int(17), int(14), int(14), int(46), int(0), time.UTC).In(local), + Latitude: 0, + Longitude: 0, + Altitude: 0, + + Trashed: false, + Archived: false, + FromPartner: false, + Favorite: false, + + FSys: videos, + FileSize: 4, + }, + } + + // Test Browse + gotLafs := []*browser.LocalAssetFile{} + assetChan := to.Browse(ctx) +assetLoop: + for { + laf, ok := <-assetChan; + if !ok { + break assetLoop; + } + gotLafs = append(gotLafs, laf) + } + + if len(gotLafs) != len(wantLafs) { + t.Errorf("Browse returned %d LocalAssetFiles instead of %d", len(gotLafs), len(wantLafs)) + } + for i, _ := range gotLafs { + // The order of the playlists in the data we read depends on + // the order of the playlists in playlists.csv, which seems to + // be random. Also we don't really care in the first place, + // so just make it predictable for the test: + sort.Sort(LocalAlbumsByName(gotLafs[i].Albums)) + sort.Sort(LocalAlbumsByName(wantLafs[i].Albums)) + + if !reflect.DeepEqual(gotLafs[i], wantLafs[i]) { + want_json, _ := json.MarshalIndent(wantLafs[i], "", " ") + got_json, _ := json.MarshalIndent(gotLafs[i], "", " ") + t.Fatalf("Prepare returned\n%s\ninstead of\n%s\nfor index %d", got_json, want_json, i) + } + } +} From 0ca29444d8543bd838f587bb166deaab35c358fa Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Fri, 28 Jun 2024 22:11:02 -0400 Subject: [PATCH 03/13] removes unnecessary unicode normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This actually breaks stuff, particularly the name of the "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص-videos.csv" playlist, which becomes "Z͔̀ͧ̑̓À͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص-videos.csv". Who doesn't love Unicode? --- browser/yt/csv.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/browser/yt/csv.go b/browser/yt/csv.go index 036d94c1..e8a6fefa 100644 --- a/browser/yt/csv.go +++ b/browser/yt/csv.go @@ -1,7 +1,6 @@ package yt import ( - "golang.org/x/text/unicode/norm" "strings" "time" "unicode/utf16" @@ -133,12 +132,6 @@ func (ytmd YouTubePlaylist) Filename() string { title = title + "-videos" - // I'm not actually sure if this is necessary but I've had enough - // of dealing with this. Hopefully YouTube is already outputting - // normalized Unicode, but there's no reason to have any faith in - // them. - title = norm.NFC.String(title) - runes := []rune(title) if len(utf16.Encode(runes)) > 47 { // Truncate the string until it's <= 47 code units @@ -178,12 +171,6 @@ func (ytv YouTubeVideo) Filename() string { title = strings.ReplaceAll(title, "*", "_") title = strings.ReplaceAll(title, "\"", "_") - // I'm not actually sure if this is necessary but I've had enough - // of dealing with this. Hopefully YouTube is already outputting - // normalized Unicode, but there's no reason to have any faith in - // them. - title = norm.NFC.String(title) - runes := []rune(title) if len(utf16.Encode(runes)) > 43 { // Truncate the string until it's <= 43 code units From b0a50a05db49c448e2a166e79e1e9076ef81a346 Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Sat, 29 Jun 2024 07:58:30 -0400 Subject: [PATCH 04/13] video title needs file suffix --- browser/yt/youtube.go | 3 ++- browser/yt/youtube_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/browser/yt/youtube.go b/browser/yt/youtube.go index cc7ab5b4..a85b5d1a 100644 --- a/browser/yt/youtube.go +++ b/browser/yt/youtube.go @@ -3,6 +3,7 @@ package yt import ( "context" "io/fs" + "path" "strconv" "github.com/simulot/immich-go/browser" @@ -169,7 +170,7 @@ func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { a := browser.LocalAssetFile{ FileName: video.Filename, - Title: video.Video.Title, + Title: video.Video.Title + path.Ext(video.Filename), Description: video.Video.Description, Albums: albums, diff --git a/browser/yt/youtube_test.go b/browser/yt/youtube_test.go index 90a564c0..a95e7148 100644 --- a/browser/yt/youtube_test.go +++ b/browser/yt/youtube_test.go @@ -459,7 +459,7 @@ func TestPrepareAndBrowse(t *testing.T) { wantLafs := []*browser.LocalAssetFile{ &browser.LocalAssetFile{ FileName: "Serenade #2.mp4", - Title: "Serenade #2", + Title: "Serenade #2.mp4", Description: "A description of Serenade #2", Albums: []browser.LocalAlbum{ browser.LocalAlbum{ @@ -509,7 +509,7 @@ func TestPrepareAndBrowse(t *testing.T) { }, &browser.LocalAssetFile{ FileName: "Serenade #1.mp4", - Title: "Serenade #1", + Title: "Serenade #1.mp4", Description: "", Albums: []browser.LocalAlbum{ browser.LocalAlbum{ @@ -564,7 +564,7 @@ func TestPrepareAndBrowse(t *testing.T) { }, &browser.LocalAssetFile{ FileName: "I manually set the location.mp4", - Title: "I manually set the location", + Title: "I manually set the location.mp4", Description: "", Albums: []browser.LocalAlbum{ browser.LocalAlbum{ @@ -604,7 +604,7 @@ func TestPrepareAndBrowse(t *testing.T) { }, &browser.LocalAssetFile{ FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋.mp4", - Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", Description: "", Albums: []browser.LocalAlbum{ browser.LocalAlbum{ @@ -639,7 +639,7 @@ func TestPrepareAndBrowse(t *testing.T) { }, &browser.LocalAssetFile{ FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋(1).mp4", - Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", Description: "A description of a Short video.", Albums: []browser.LocalAlbum{ browser.LocalAlbum{ @@ -679,7 +679,7 @@ func TestPrepareAndBrowse(t *testing.T) { }, &browser.LocalAssetFile{ FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋(2).mp4", - Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", Description: "", Albums: []browser.LocalAlbum{ browser.LocalAlbum{ From 5fbe596bf649aa771b04d18cad3358babcd959dc Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Sat, 29 Jun 2024 08:20:41 -0400 Subject: [PATCH 05/13] makes `upload -youtube` actually work! --- cmd/upload/upload.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/cmd/upload/upload.go b/cmd/upload/upload.go index 0f1ddbdc..0dd55da8 100644 --- a/cmd/upload/upload.go +++ b/cmd/upload/upload.go @@ -20,6 +20,7 @@ import ( "github.com/simulot/immich-go/browser" "github.com/simulot/immich-go/browser/files" "github.com/simulot/immich-go/browser/gp" + "github.com/simulot/immich-go/browser/yt" "github.com/simulot/immich-go/cmd" "github.com/simulot/immich-go/helpers/fileevent" "github.com/simulot/immich-go/helpers/fshelper" @@ -36,6 +37,7 @@ type UpCmd struct { fsyss []fs.FS // pseudo file system to browse GooglePhotos bool // For reading Google Photos takeout files + YouTube bool // For reading YouTube takeout files Delete bool // Delete original file after import CreateAlbumAfterFolder bool // Create albums for assets based on the parent folder or a given name ImportIntoAlbum string // All assets will be added to this album @@ -107,9 +109,13 @@ func newCommand(ctx context.Context, common *cmd.SharedFlags, args []string) (*U "google-photos", "Import GooglePhotos takeout zip files", myflag.BoolFlagFn(&app.GooglePhotos, false)) + cmd.BoolFunc( + "youtube", + "Import YouTube takeout zip files", + myflag.BoolFlagFn(&app.YouTube, false)) cmd.BoolFunc( "create-albums", - " google-photos only: Create albums like there were in the source (default: TRUE)", + " google-photos/youtube only: Create albums like there were in the source (default: TRUE)", myflag.BoolFlagFn(&app.CreateAlbums, true)) cmd.StringVar(&app.PartnerAlbum, "partner-album", @@ -198,6 +204,9 @@ func (app *UpCmd) run(ctx context.Context) error { case app.GooglePhotos: app.Log.Info("Browsing google take out archive...") app.browser, err = app.ReadGoogleTakeOut(ctx, app.fsyss) + case app.YouTube: + app.Log.Info("Browsing youtube take out archive...") + app.browser, err = app.ReadYouTubeTakeOut(ctx, app.fsyss) default: app.Log.Info("Browsing folder(s)...") app.browser, err = app.ExploreLocalFolder(ctx, app.fsyss) @@ -683,14 +692,15 @@ func (app *UpCmd) handleAsset(ctx context.Context, a *browser.LocalAssetFile) er if app.ImportIntoAlbum != "" || (app.GooglePhotos && (app.CreateAlbums || app.PartnerAlbum != "")) || - (!app.GooglePhotos && app.CreateAlbumAfterFolder) { + (!app.GooglePhotos && app.CreateAlbumAfterFolder) || + (app.YouTube && app.CreateAlbums) { albums := []browser.LocalAlbum{} if app.ImportIntoAlbum != "" { albums = append(albums, browser.LocalAlbum{Path: app.ImportIntoAlbum, Name: app.ImportIntoAlbum}) } else { switch { - case app.GooglePhotos: + case app.GooglePhotos || app.YouTube: albums = append(albums, a.Albums...) if app.PartnerAlbum != "" && a.FromPartner { albums = append(albums, browser.LocalAlbum{Path: app.PartnerAlbum, Name: app.PartnerAlbum}) @@ -750,6 +760,11 @@ func (app *UpCmd) ReadGoogleTakeOut(ctx context.Context, fsyss []fs.FS) (browser return gp.NewTakeout(ctx, app.Jnl, app.Immich.SupportedMedia(), fsyss...) } +func (app *UpCmd) ReadYouTubeTakeOut(ctx context.Context, fsyss []fs.FS) (browser.Browser, error) { + app.Delete = false + return yt.NewTakeout(ctx, app.Jnl, app.Immich.SupportedMedia(), fsyss...) +} + func (app *UpCmd) ExploreLocalFolder(ctx context.Context, fsyss []fs.FS) (browser.Browser, error) { b, err := files.NewLocalFiles(ctx, app.Jnl, fsyss...) if err != nil { From 756d80f073679fdff41a89131a65b311cd6c6491 Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Sat, 29 Jun 2024 17:38:30 -0400 Subject: [PATCH 06/13] adds some logging and error reporting --- browser/yt/youtube.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/browser/yt/youtube.go b/browser/yt/youtube.go index a85b5d1a..29cbf24e 100644 --- a/browser/yt/youtube.go +++ b/browser/yt/youtube.go @@ -88,9 +88,11 @@ func (to *Takeout) Prepare(ctx context.Context) error { for i, _ := range ytplaylists { playlist := ytplaylists[i] if playlist.Title == "Watch later" { + to.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, "Watch later.csv", "reason", "useless file") continue } + to.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, playlist.Filename(), "reason", "playlist file referenced in playlists.csv") ytplaylistvideos, err := fshelper.ReadCSV[YouTubePlaylistVideo](pfs, playlist.Filename()) if err != nil { return err @@ -132,6 +134,8 @@ func (to *Takeout) Prepare(ctx context.Context) error { } filename += ".mp4" + to.log.Record(ctx, fileevent.DiscoveredVideo, nil, filename) + synth := SynthesizedYouTubeVideo{ Channel: channels[video.ChannelID], Playlists: playlists[video.VideoID], @@ -156,6 +160,7 @@ func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { for _, video := range to.videos { fileinfo, err := fs.Stat(video.Fsys, video.Filename) if err != nil { + assetChan <- &browser.LocalAssetFile{Err: err} continue } From 21d6ee6ce6217a61f024a737d6c4d0606b056afb Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Sat, 29 Jun 2024 18:00:46 -0400 Subject: [PATCH 07/13] more logging and complaining about albums --- browser/yt/youtube.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/browser/yt/youtube.go b/browser/yt/youtube.go index 29cbf24e..75c0008c 100644 --- a/browser/yt/youtube.go +++ b/browser/yt/youtube.go @@ -82,6 +82,7 @@ func (to *Takeout) Prepare(ctx context.Context) error { // VideoID => YouTubePlaylist playlists := map[string][]*YouTubePlaylist{} ytplaylists, err := fshelper.ReadCSV[YouTubePlaylist](pfs, "playlists.csv") + playlistTitlesToIDs := map[string]string{} if err != nil { return err } @@ -91,6 +92,12 @@ func (to *Takeout) Prepare(ctx context.Context) error { to.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, "Watch later.csv", "reason", "useless file") continue } + playlistID, ok := playlistTitlesToIDs[playlist.Title] + if !ok { + playlistTitlesToIDs[playlist.Title] = playlist.PlaylistID + } else { + to.log.Record(ctx, fileevent.AnalysisLocalDuplicate, nil, "playlists.csv", "duplicate playlist name", "Playlist IDs " + playlistID + " and " + playlist.PlaylistID + " are both named '" + playlist.Title + "'; there's no way of knowing which playlist is actually being stored") + } to.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, playlist.Filename(), "reason", "playlist file referenced in playlists.csv") ytplaylistvideos, err := fshelper.ReadCSV[YouTubePlaylistVideo](pfs, playlist.Filename()) @@ -166,6 +173,17 @@ func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { albums := []browser.LocalAlbum{} for _, playlist := range video.Playlists { + // Immich albums support having a description, + // and we have a description of each playlist + // from playlists.csv, but immich-go doesn't + // support passing those descriptions through + // without many changes: UpCmd.updateAlbums is + // a map from album name to a list of assets, + // and UpCmd.AddToAlbum would need to have the + // description added to its many calls. Or + // we'd just need a new way to provide album + // description and I don't understand the code + // well enough to undertake that. album := browser.LocalAlbum{ Path: playlist.Title, Name: playlist.Title, From 8e899e1234fb55c9d785931cff029bd99e5327be Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Sat, 29 Jun 2024 18:15:48 -0400 Subject: [PATCH 08/13] add album for channel itself --- browser/yt/youtube.go | 7 ++++++- browser/yt/youtube_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/browser/yt/youtube.go b/browser/yt/youtube.go index 75c0008c..c1f446b6 100644 --- a/browser/yt/youtube.go +++ b/browser/yt/youtube.go @@ -171,7 +171,12 @@ func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { continue } - albums := []browser.LocalAlbum{} + albums := []browser.LocalAlbum{ + browser.LocalAlbum{ + Path: video.Channel.Title + "'s YouTube channel", + Name: video.Channel.Title + "'s YouTube channel", + }, + } for _, playlist := range video.Playlists { // Immich albums support having a description, // and we have a description of each playlist diff --git a/browser/yt/youtube_test.go b/browser/yt/youtube_test.go index a95e7148..07d650d5 100644 --- a/browser/yt/youtube_test.go +++ b/browser/yt/youtube_test.go @@ -456,12 +456,17 @@ func TestPrepareAndBrowse(t *testing.T) { local, _ := tzone.Local() + channelAlbum := browser.LocalAlbum{ + Path: "Jonathan Stafford's YouTube channel", + Name: "Jonathan Stafford's YouTube channel", + } wantLafs := []*browser.LocalAssetFile{ &browser.LocalAssetFile{ FileName: "Serenade #2.mp4", Title: "Serenade #2.mp4", Description: "A description of Serenade #2", Albums: []browser.LocalAlbum{ + channelAlbum, browser.LocalAlbum{ //Path: "A playlist-videos.csv", Path: "A playlist", @@ -512,6 +517,7 @@ func TestPrepareAndBrowse(t *testing.T) { Title: "Serenade #1.mp4", Description: "", Albums: []browser.LocalAlbum{ + channelAlbum, browser.LocalAlbum{ //Path: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘.csv" Path: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊tema🙋tis🙌rolod🙍muspi🙎meroL🙏", @@ -567,6 +573,7 @@ func TestPrepareAndBrowse(t *testing.T) { Title: "I manually set the location.mp4", Description: "", Albums: []browser.LocalAlbum{ + channelAlbum, browser.LocalAlbum{ //Path: "`-=[]_,._~!@#$_^&_()_+{}_-videos.csv" Path: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"?", @@ -607,6 +614,7 @@ func TestPrepareAndBrowse(t *testing.T) { Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", Description: "", Albums: []browser.LocalAlbum{ + channelAlbum, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", @@ -642,6 +650,7 @@ func TestPrepareAndBrowse(t *testing.T) { Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", Description: "A description of a Short video.", Albums: []browser.LocalAlbum{ + channelAlbum, browser.LocalAlbum{ //Path: "A playlist-videos.csv" Path: "A playlist", @@ -682,6 +691,7 @@ func TestPrepareAndBrowse(t *testing.T) { Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", Description: "", Albums: []browser.LocalAlbum{ + channelAlbum, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", From 61b87c12c1eb30c2884b75921646ee15fb019df5 Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Sat, 29 Jun 2024 18:44:04 -0400 Subject: [PATCH 09/13] fixes video description Fixes video description to be the title or the description or both, combined sensibly. --- browser/yt/youtube.go | 10 +++++++++- browser/yt/youtube_test.go | 12 ++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/browser/yt/youtube.go b/browser/yt/youtube.go index c1f446b6..cae5b0a1 100644 --- a/browser/yt/youtube.go +++ b/browser/yt/youtube.go @@ -196,10 +196,18 @@ func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { albums = append(albums, album) } + description, title_ok := video.Video.CleanTitle() + desc, desc_ok := video.Video.CleanDescription() + if title_ok && desc_ok { + description += "\n\n" + desc + } else if !title_ok { + description = desc + } + a := browser.LocalAssetFile{ FileName: video.Filename, Title: video.Video.Title + path.Ext(video.Filename), - Description: video.Video.Description, + Description: description, Albums: albums, DateTaken: video.Video.Time(), diff --git a/browser/yt/youtube_test.go b/browser/yt/youtube_test.go index 07d650d5..6c772937 100644 --- a/browser/yt/youtube_test.go +++ b/browser/yt/youtube_test.go @@ -464,7 +464,7 @@ func TestPrepareAndBrowse(t *testing.T) { &browser.LocalAssetFile{ FileName: "Serenade #2.mp4", Title: "Serenade #2.mp4", - Description: "A description of Serenade #2", + Description: "Serenade #2\n\nA description of Serenade #2", Albums: []browser.LocalAlbum{ channelAlbum, browser.LocalAlbum{ @@ -515,7 +515,7 @@ func TestPrepareAndBrowse(t *testing.T) { &browser.LocalAssetFile{ FileName: "Serenade #1.mp4", Title: "Serenade #1.mp4", - Description: "", + Description: "Serenade #1", Albums: []browser.LocalAlbum{ channelAlbum, browser.LocalAlbum{ @@ -571,7 +571,7 @@ func TestPrepareAndBrowse(t *testing.T) { &browser.LocalAssetFile{ FileName: "I manually set the location.mp4", Title: "I manually set the location.mp4", - Description: "", + Description: "I manually set the location", Albums: []browser.LocalAlbum{ channelAlbum, browser.LocalAlbum{ @@ -612,7 +612,7 @@ func TestPrepareAndBrowse(t *testing.T) { &browser.LocalAssetFile{ FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋.mp4", Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", - Description: "", + Description: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", Albums: []browser.LocalAlbum{ channelAlbum, browser.LocalAlbum{ @@ -648,7 +648,7 @@ func TestPrepareAndBrowse(t *testing.T) { &browser.LocalAssetFile{ FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋(1).mp4", Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", - Description: "A description of a Short video.", + Description: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص\n\nA description of a Short video.", Albums: []browser.LocalAlbum{ channelAlbum, browser.LocalAlbum{ @@ -689,7 +689,7 @@ func TestPrepareAndBrowse(t *testing.T) { &browser.LocalAssetFile{ FileName: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋(2).mp4", Title: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص.mp4", - Description: "", + Description: "`-=[]\\;',./~!@#$%^\u0026*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", Albums: []browser.LocalAlbum{ channelAlbum, browser.LocalAlbum{ From 6d8868b2418b64591c47871f59cdcb96405bd10d Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Sun, 30 Jun 2024 09:16:43 -0400 Subject: [PATCH 10/13] improves matching video files to titles Not all videos have a .mp4 extension --- .../playlists/Favorites-videos.csv | 7 ++ .../playlists/playlists.csv | 1 + browser/yt/csv.go | 16 ++- browser/yt/csv_test.go | 31 +++++- browser/yt/youtube.go | 101 ++++++++++++++++-- browser/yt/youtube_test.go | 2 +- 6 files changed, 140 insertions(+), 18 deletions(-) create mode 100644 browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/Favorites-videos.csv diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/Favorites-videos.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/Favorites-videos.csv new file mode 100644 index 00000000..eab9e2a6 --- /dev/null +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/Favorites-videos.csv @@ -0,0 +1,7 @@ +Video ID,Playlist Video Creation Timestamp +rl1vcIiguJV,2017-04-10T12:19:55+00:00 + + + + + diff --git a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/playlists.csv b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/playlists.csv index 2388d68c..47c5ac22 100644 --- a/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/playlists.csv +++ b/browser/yt/TEST_DATA/20240623T224719/Takeout/YouTube and YouTube Music/playlists/playlists.csv @@ -16,6 +16,7 @@ yojzPdFgHjBNXsUcmTsmo6g1hTqsIkZUSB,False,,My very long playlist title 0123456789 ykzP/AtWUAgtD6kfpZyk+CCipFNPlh27FA,False,,"`-=[]\;',./~!@#$%^&*()_+{}|:""?",,2023-12-17T13:57:31+00:00,2023-12-17T14:32:42+00:00,Manual,Private /63ek5JSj2ZcQSXaBiTslzKRSb+kK015UI,False,This is my playlist,A playlist,,2023-12-15T01:47:18+00:00,2023-12-15T01:48:39+00:00,Manual,Private ldyZEf/SuTx9DgVls+WTopx7BG8Ufi6kl4,False,,Watch later,en_US,2015-03-22T06:43:50+00:00,2024-05-10T01:25:36+00:00,Manual,Private +dYZq/4FvOYqqz2yKN/8TbPRC10+5ibTc81,True,,Favorites,en_US,2012-01-29T08:43:21+00:00,2024-05-10T01:25:36+00:00,Manual,Private diff --git a/browser/yt/csv.go b/browser/yt/csv.go index e8a6fefa..d332ada1 100644 --- a/browser/yt/csv.go +++ b/browser/yt/csv.go @@ -148,11 +148,14 @@ func (ytmd YouTubePlaylist) Filename() string { return title + ".csv" } -func (ytv YouTubeVideo) Filename() string { - // This is identical to YouTubePlaylist.Filename() except that: +func (ytv YouTubeVideo) Glob() (string, bool) { + // This is largely identical to YouTubePlaylist.Filename() except that: // 1. The length is different!? // 2. No counter is added by this function - // 3. No file extension is added by this function + // 3. No file extension is added to this function, partly becaues its + // unknown, and partly because we will need to append the counter + // 4. This function generates an escaped glob for use with fs.Glob + // (or path.Match) // // All of the proscriptions of YouTubePlaylist.Filename() apply here // as well @@ -171,6 +174,7 @@ func (ytv YouTubeVideo) Filename() string { title = strings.ReplaceAll(title, "*", "_") title = strings.ReplaceAll(title, "\"", "_") + original := title runes := []rune(title) if len(utf16.Encode(runes)) > 43 { // Truncate the string until it's <= 43 code units @@ -183,9 +187,13 @@ func (ytv YouTubeVideo) Filename() string { } title = string(runes) } + full := original == title - return title + title = strings.ReplaceAll(title, "[", "\\[") + + return title, full } + func (ytv YouTubeVideo) CleanTitle() (string, bool) { title := strings.Trim(ytv.Title, " ") if title != "" { diff --git a/browser/yt/csv_test.go b/browser/yt/csv_test.go index 7b3de935..ff7f0289 100644 --- a/browser/yt/csv_test.go +++ b/browser/yt/csv_test.go @@ -213,6 +213,16 @@ func TestReadYouTubePlaylist(t *testing.T) { VideoOrder: "Manual", Visibility: "Private", }, + yt.YouTubePlaylist{ + PlaylistID: "dYZq/4FvOYqqz2yKN/8TbPRC10+5ibTc81", + Description: "", + Title: "Favorites", + TitleLanguage: "en_US", + CreateTimestamp: "2012-01-29T08:43:21+00:00", + UpdateTimestamp: "2024-05-10T01:25:36+00:00", + VideoOrder: "Manual", + Visibility: "Private", + }, } var got []yt.YouTubePlaylist @@ -496,7 +506,7 @@ func TestPlaylistFilename(t *testing.T) { } for i, playlist := range playlists { - if playlist.Title != "Watch later" { + if playlist.Title != "Watch later" && playlist.Title != "Favorites" { filename := playlist.Filename() _, err := fs.Stat(dir, filename) if err != nil { @@ -506,26 +516,36 @@ func TestPlaylistFilename(t *testing.T) { } } -func TestVideoFilename(t *testing.T) { +func TestVideoGlob(t *testing.T) { testCases := []struct { title string expected string + full bool }{ { title: "A description of Serenade #2", expected: "A description of Serenade #2", + full: true, }, { title: "Serenade #1", expected: "Serenade #1", + full: true, }, { title: "I manually set the location", expected: "I manually set the location", + full: true, }, { title: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"? 👱🏻🧟‍♀👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷 Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", - expected: "`-=[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋", + expected: "`-=\\[]_,._~!@#$_^&_()_+{}_ 👱🏻🧟‍♀👚‍❀‍💋", + full: false, + }, + { + title: "IMG_0253[1].MOV", + expected: "IMG_0253\\[1].MOV", + full: true, }, } @@ -533,10 +553,13 @@ func TestVideoFilename(t *testing.T) { sut := yt.YouTubeVideo{ Title: tc.title, } - filename := sut.Filename() + filename, full := sut.Glob() if filename != tc.expected { t.Errorf("Got\n%s\ninstead of\n%s\nfrom\n%s", filename, tc.expected, tc.title) } + if full != tc.full { + t.Errorf("Got full:%t instead of %t for %s", full, tc.full, tc.title) + } } } diff --git a/browser/yt/youtube.go b/browser/yt/youtube.go index cae5b0a1..e65c6724 100644 --- a/browser/yt/youtube.go +++ b/browser/yt/youtube.go @@ -2,9 +2,12 @@ package yt import ( "context" + "fmt" "io/fs" "path" + "regexp" "strconv" + "strings" "github.com/simulot/immich-go/browser" "github.com/simulot/immich-go/helpers/fileevent" @@ -24,6 +27,7 @@ type SynthesizedYouTubeVideo struct { type Takeout struct { fsyss []fs.FS videos []*SynthesizedYouTubeVideo + faves map[string]bool log *fileevent.Recorder sm immich.SupportedMedia } @@ -32,6 +36,7 @@ func NewTakeout(ctx context.Context, l *fileevent.Recorder, sm immich.SupportedM to := Takeout{ fsyss: fsyss, videos: []*SynthesizedYouTubeVideo{}, + faves: map[string]bool{}, log: l, sm: sm, } @@ -42,6 +47,15 @@ func NewTakeout(ctx context.Context, l *fileevent.Recorder, sm immich.SupportedM // Prepare scans all files to build gather and aggregate the metadata func (to *Takeout) Prepare(ctx context.Context) error { + smExts := []string{} + for ext := range to.sm { + if to.sm[ext] == immich.TypeVideo { + smExts = append(smExts, ext[1:]) + } + } + pattern := "\\(\\d+\\)\\.(?:" + strings.Join(smExts, "|") + ")" + re := regexp.MustCompile(pattern) + for _, fsys := range to.fsyss { tofs, err := fs.Sub(fsys, "Takeout") if err != nil { @@ -86,11 +100,16 @@ func (to *Takeout) Prepare(ctx context.Context) error { if err != nil { return err } + hasFavorites := false for i, _ := range ytplaylists { playlist := ytplaylists[i] if playlist.Title == "Watch later" { to.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, "Watch later.csv", "reason", "useless file") continue + } else if playlist.Title == "Favorites" { + to.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, "Favorites-videos.csv", "reason", "playlist file referenced in playlists.csv") + hasFavorites = true + continue } playlistID, ok := playlistTitlesToIDs[playlist.Title] if !ok { @@ -110,6 +129,17 @@ func (to *Takeout) Prepare(ctx context.Context) error { } } + if hasFavorites { + favoriteVideos, err := fshelper.ReadCSV[YouTubePlaylistVideo](pfs, "Favorites-videos.csv") + if err != nil { + return err + } + for i, _ := range favoriteVideos { + playlistvideo := favoriteVideos[i] + to.faves[playlistvideo.VideoID] = true + } + } + // VideoID => YouTubeVideoRecording recordings := map[string]*YouTubeVideoRecording{} ytrecordings, err := fshelper.ReadCSV[YouTubeVideoRecording](vmfs, "video recordings.csv") @@ -131,17 +161,68 @@ func (to *Takeout) Prepare(ctx context.Context) error { for i, _ := range videos { video := videos[i] - filename := video.Filename() - count, ok := filenames[filename] - if !ok { - filenames[filename] = 1 + glob, full := video.Glob() + + // XXX Haven't tested if counts are per basename or + // XXX include the file extension, i.e., is it: + // title.mp4 and title(1).mp4, but title.avi + // or + // title.mp4 and title(1).mp4 and title(2).avi + count, count_ok := filenames[glob] + if !count_ok { + filenames[glob] = 1 } else { - filenames[filename] = count + 1 - filename = filename + "(" + strconv.Itoa(count) + ")" + filenames[glob] = count + 1 + glob += "(" + strconv.Itoa(count) + ")" } - filename += ".mp4" - to.log.Record(ctx, fileevent.DiscoveredVideo, nil, filename) + if full { + glob += "." + } + glob += "*" + + filenames, err := fs.Glob(vfs, glob) + if err != nil { + to.log.Record(ctx, fileevent.Error, nil, glob, "reason", "no matching files found") + continue + } else if len(filenames) != 1{ + if !count_ok { + // This is the first instance of this + // glob, so ignore all the matches that + // include a counter. This is only + // really a problem when !full as well, + // but there could always be a . in the + // filename so not including that in + // the conditional. + uncountedFilenames := []string{} + for i, _ := range filenames { + if !re.MatchString(filenames[i]) { + uncountedFilenames = append(uncountedFilenames, filenames[i]) + } + } + + if len(uncountedFilenames) == 1 { + filenames = uncountedFilenames + } + } + + if len(filenames) != 1 { + to.log.Record(ctx, fileevent.Error, nil, glob, "reason", fmt.Sprintf("%d matching files found", len(filenames))) + continue + } + } + + filename := filenames[0] + ext := strings.ToLower(path.Ext(filename)) + switch to.sm.TypeFromExt(ext) { + case immich.TypeImage: + to.log.Record(ctx, fileevent.DiscoveredImage, nil, filename) + case immich.TypeVideo: + to.log.Record(ctx, fileevent.DiscoveredVideo, nil, filename) + case immich.TypeUnknown: + to.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, filename, "reason", "unsupported file type") + continue + } synth := SynthesizedYouTubeVideo{ Channel: channels[video.ChannelID], @@ -204,6 +285,8 @@ func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { description = desc } + _, favorite := to.faves[video.Video.VideoID] + a := browser.LocalAssetFile{ FileName: video.Filename, Title: video.Video.Title + path.Ext(video.Filename), @@ -218,7 +301,7 @@ func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { Trashed: false, Archived: false, FromPartner: false, - Favorite: false, + Favorite: favorite, FSys: video.Fsys, FileSize: int(fileinfo.Size()), diff --git a/browser/yt/youtube_test.go b/browser/yt/youtube_test.go index 6c772937..41c49e9d 100644 --- a/browser/yt/youtube_test.go +++ b/browser/yt/youtube_test.go @@ -563,7 +563,7 @@ func TestPrepareAndBrowse(t *testing.T) { Trashed: false, Archived: false, FromPartner: false, - Favorite: false, + Favorite: true, FSys: videos, FileSize: 6, From 5313469a2db19c3b07b05958d34ba308de2040fa Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Thu, 4 Jul 2024 14:31:16 -0400 Subject: [PATCH 11/13] adds readme and youtube docs --- docs/youtube-takeout.md | 24 ++++++++++++++++++++++++ readme.md | 16 ++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 docs/youtube-takeout.md diff --git a/docs/youtube-takeout.md b/docs/youtube-takeout.md new file mode 100644 index 00000000..dadae91b --- /dev/null +++ b/docs/youtube-takeout.md @@ -0,0 +1,24 @@ +# The YouTube Takeout case +Most of the flaws in the YouTube Takeout data are probably a result of optimizing for "normal" users dealing with the data, rather than machines. Thankfully, it's largely possible to accurately and automatically import data from a YouTube Takeout into another program, except for the shortcomings noted below. + +# Shortcomings + +## Playlists with similar names + +YouTube's algorithm serializes playlists to files uses the first ~47 characters of the playlist name as the stem of the filename. If two playlists have the same 47 characters in common, only one of them is serialized to the playlist file, and the other is lost, and there is no way of knowing which one is which. + +## Videos with similar names and/or file extensions +YouTube's algorithm for serializing videos to files uses the first ~43 characters of the video name as the stem of the filename. After those 43 characters, a one-up counter is appended to the name, e.g., `(1)`. It is not clear if the file the provides the video metadata nd the filename counters are synched, i.e., is the first video named "Example" written to `Example.mp4`, the second video named "Example" written to `Example(1).mp4`, etc. + +Additionally, the file that provides video metadata does not provide any information about the video's file extension, but YouTube Takeout videos can have multiple file extensions. If the Takeout includes both `video.mp4` and `video.wmv` and `video.avi` it is highly likely that the video file and metadata will not be paired correctly. + +## Missing videos +YouTube Takeout sometimes fails to include individual video files for unknown reasons. The metadata appears to still be present but the video itself is mssing. In this case `immich-go` will report an error about the missing video file. + +## Playlist descriptions +Both the YouTube channel and and playlists are turned into Immich albums. Both YouTube channels and playlists can include descriptions, but `immich-go` does not currently support adding these descriptions to albums, so they are lost. + +Video descriptions are preserved. + +# What if you have problems with a takeout archive? +Please open an issue with details. Tag the issue with `@thecabinet`. diff --git a/readme.md b/readme.md index 6dc18b07..cf46e5f9 100644 --- a/readme.md +++ b/readme.md @@ -18,14 +18,14 @@ ValidateConnection, GET, http://your-immich-server:2283/api/user/me, 404 Not Fou ## Key Features: -* **Effortlessly Upload Large Google Photos Takeouts:** Immich-Go excels at handling the massive archives you download from Google Photos using Google Takeout. It efficiently processes these archives while preserving valuable metadata like GPS location, date taken, and album information. +* **Effortlessly Upload Large Google Takeouts:** Immich-Go excels at handling the massive archives you download from Google Photos and YouTube using Google Takeout. It efficiently processes these archives while preserving valuable metadata like GPS location, date taken, and album information. * **Flexible Uploads:** Immich-Go isn't limited to Google Photos. You can upload photos directly from your computer folders, folders tree and ZIP archives. * **Simple Installation:** Immich-Go doesn't require NodeJS or Docker for installation. This makes it easy to get started, even for those less familiar with technical environments. * **Prioritize Quality:** Immich-Go discards any lower-resolution versions that might be included in Google Photos Takeout, ensuring you have the best possible copies on your Immich server. * **Stack burst and raw/jpg photos**: Group together related photos in Immich. -## Google Photos Best Practices: +## Google Photos and YouTube Best Practices: * **Taking Out Your Photos:** * Choose the ZIP format when creating your takeout for easier import. @@ -135,6 +135,18 @@ Specialized options for Google Photos management: Read [here](docs/google-takeout.md) to understand how Google Photos takeout isn't easy to handle. +### YouTube options: +Specialized options for YouTube management: + +| **Parameter** | **Description** | **Default value** | +| ---------------------------------- | -------------------------------------------------------------------------- | ----------------- | +| `-youtube` | Import from a YouTube structured archive, recreating corresponding albums. | | +| `-create-albums` | Controls creation of YouTube albums in Immich. | `TRUE` | + +`-create-albums` will create an album for the YouTube channel and each playlist. + +Read [here](docs/youtube-takeout.md) to understand how YouTube takeout isn't easy to handle and some shortcomings of the YouTube import process. + ### How date of photos is determined From ee07ea4f330d8b458bf1fea6fec56cb345dafca6 Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Mon, 15 Jul 2024 09:17:05 -0400 Subject: [PATCH 12/13] makes playlist description into album description --- browser/yt/youtube.go | 12 +----------- browser/yt/youtube_test.go | 27 +++++++++++++++++++++++++++ docs/youtube-takeout.md | 5 ----- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/browser/yt/youtube.go b/browser/yt/youtube.go index 98abfd8a..023dafd2 100644 --- a/browser/yt/youtube.go +++ b/browser/yt/youtube.go @@ -277,20 +277,10 @@ func (to *Takeout) Browse(ctx context.Context) chan *browser.LocalAssetFile { }, } for _, playlist := range video.Playlists { - // Immich albums support having a description, - // and we have a description of each playlist - // from playlists.csv, but immich-go doesn't - // support passing those descriptions through - // without many changes: UpCmd.updateAlbums is - // a map from album name to a list of assets, - // and UpCmd.AddToAlbum would need to have the - // description added to its many calls. Or - // we'd just need a new way to provide album - // description and I don't understand the code - // well enough to undertake that. album := browser.LocalAlbum{ Path: playlist.Title, Title: playlist.Title, + Description: playlist.Description, } albums = append(albums, album) } diff --git a/browser/yt/youtube_test.go b/browser/yt/youtube_test.go index fc83e4b3..7722f7ab 100644 --- a/browser/yt/youtube_test.go +++ b/browser/yt/youtube_test.go @@ -471,31 +471,37 @@ func TestPrepareAndBrowse(t *testing.T) { //Path: "A playlist-videos.csv", Path: "A playlist", Title: "A playlist", + Description: "This is my playlist", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv", Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv", Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv", Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My very long playlist title 0123456789 ABCDEFGH.csv", Path: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", Title: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + Description: "", }, browser.LocalAlbum{ //Path: "My very long playlist title 0123456789 ABCDEFGH.csv", Path: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", Title: "My very long playlist title 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrs", + Description: "", }, }, @@ -524,36 +530,43 @@ func TestPrepareAndBrowse(t *testing.T) { //Path: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘.csv" Path: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊tema🙋tis🙌rolod🙍muspi🙎meroL🙏", Title: "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊tema🙋tis🙌rolod🙍muspi🙎meroL🙏", + Description: "", }, browser.LocalAlbum{ //Path: "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌.csv" Path: "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", Title: "😀Lorem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + Description: "", }, browser.LocalAlbum{ //Path: "My very long playlist title 0123456789 ABCDEFG-.csv" Path: "My very long playlist title 0123456789 ABCDEFG", Title: "My very long playlist title 0123456789 ABCDEFG", + Description: "", }, browser.LocalAlbum{ //Path: "My very long playlist title 0123456789 ABCDEF-v.csv" Path: "My very long playlist title 0123456789 ABCDEF", Title: "My very long playlist title 0123456789 ABCDEF", + Description: "", }, browser.LocalAlbum{ //Path: "My very long playlist title 0123456789 ABCDE-vi.csv" Path: "My very long playlist title 0123456789 ABCDE", Title: "My very long playlist title 0123456789 ABCDE", + Description: "", }, browser.LocalAlbum{ //Path: "My very long playlist title 0123456789 ABCD-vid.csv" Path: "My very long playlist title 0123456789 ABCD", Title: "My very long playlist title 0123456789 ABCD", + Description: "", }, browser.LocalAlbum{ //Path: "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌.csv" Path: "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", Title: "😀orem😁ipsum😂dolor😃sit😄amet😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😀😥😊😧😚😩😪😫😬😭😮😯😰😱😲😳😎😵😶😷😞😹😺😻😌😜😟😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏", + Description: "", }, }, @@ -582,21 +595,25 @@ func TestPrepareAndBrowse(t *testing.T) { //Path: "`-=[]_,._~!@#$_^&_()_+{}_-videos.csv" Path: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"?", Title: "`-=[]\\;',./~!@#$%^&*()_+{}|:\"?", + Description: "", }, browser.LocalAlbum{ //Path: "A playlist-videos.csv" Path: "A playlist", Title: "A playlist", + Description: "This is my playlist", }, browser.LocalAlbum{ //Path: "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩.csv" Path: "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷", Title: "👱👱🏻👱🏌👱🏜👱🏟👱🏿 🧟‍♀🧟‍♂ 👚‍❀‍💋‍👚👩‍👩‍👧‍👊🏳‍⚧🇵🇷", + Description: "", }, browser.LocalAlbum{ //Path: "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص-videos.csv" Path: "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", Title: "Z͔ͧ̑̓̀ä͖̭̈̇lͮ̒ͫǧ̗͚̚o̙̔ͮ̇͐̇ اختؚار النص", + Description: "", }, }, @@ -625,16 +642,19 @@ func TestPrepareAndBrowse(t *testing.T) { //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, }, @@ -663,21 +683,25 @@ func TestPrepareAndBrowse(t *testing.T) { //Path: "A playlist-videos.csv" Path: "A playlist", Title: "A playlist", + Description: "This is my playlist", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, }, @@ -706,16 +730,19 @@ func TestPrepareAndBrowse(t *testing.T) { //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, browser.LocalAlbum{ //Path: "My playlist with a duplicate name-videos.csv Path: "My playlist with a duplicate name", Title: "My playlist with a duplicate name", + Description: "", }, }, diff --git a/docs/youtube-takeout.md b/docs/youtube-takeout.md index dadae91b..383bf009 100644 --- a/docs/youtube-takeout.md +++ b/docs/youtube-takeout.md @@ -15,10 +15,5 @@ Additionally, the file that provides video metadata does not provide any informa ## Missing videos YouTube Takeout sometimes fails to include individual video files for unknown reasons. The metadata appears to still be present but the video itself is mssing. In this case `immich-go` will report an error about the missing video file. -## Playlist descriptions -Both the YouTube channel and and playlists are turned into Immich albums. Both YouTube channels and playlists can include descriptions, but `immich-go` does not currently support adding these descriptions to albums, so they are lost. - -Video descriptions are preserved. - # What if you have problems with a takeout archive? Please open an issue with details. Tag the issue with `@thecabinet`. From c65080df0286ca6cb7c33bc4f13609673517a08d Mon Sep 17 00:00:00 2001 From: Jonathan Stafford Date: Mon, 15 Jul 2024 09:58:46 -0400 Subject: [PATCH 13/13] fixes simultaneous use of -google-photos and -youtube --- cmd/upload/upload.go | 8 +++++++- readme.md | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/upload/upload.go b/cmd/upload/upload.go index c06b6abd..3013692d 100644 --- a/cmd/upload/upload.go +++ b/cmd/upload/upload.go @@ -4,6 +4,7 @@ package upload import ( "context" + "errors" "flag" "fmt" "io/fs" @@ -194,7 +195,12 @@ func newCommand(ctx context.Context, common *cmd.SharedFlags, args []string) (*U return nil, err } - app.fsyss, err = fshelper.ParsePath(cmd.Args(), app.GooglePhotos) + if app.GooglePhotos && app.YouTube { + err = errors.New("-google-photos and -youtube cannot be used simultaneously") + return nil, err + } + + app.fsyss, err = fshelper.ParsePath(cmd.Args(), app.GooglePhotos || app.YouTube) if err != nil { return nil, err } diff --git a/readme.md b/readme.md index 466e37ea..9f693b66 100644 --- a/readme.md +++ b/readme.md @@ -35,6 +35,7 @@ * For **.tgz** files (compressed tar archives), you'll need to decompress all the files into a single folder before importing. When using the import tool, don't forget the `-google-photos` option. * You can remove any unwanted files or folders from your takeout before importing. Immich-go might warn you about missing JSON files, but it should still import your photos successfully. * Restarting an interrupted import won't cause any problems and it will resume the work where it was left. + * Use `-google-photos` and `-youtube` on separate runs; you cannot import both types of data simultaneously For insights into the reasoning behind this alternative to `immich-cli`, please read the motivation [here](docs/motivation.md).