From 3bd52853f62aeea52db98ca9bf3ed1759576e800 Mon Sep 17 00:00:00 2001 From: Amadej Kastelic Date: Fri, 30 Aug 2024 11:05:39 +0200 Subject: [PATCH 1/2] Fetch comments support --- Pipfile.lock | 200 ++++++++++++++--------------- bot/common/utils.py | 17 +++ bot/domain.py | 48 +++++-- bot/downloader/base.py | 3 + bot/downloader/facebook/client.py | 4 + bot/downloader/instagram/client.py | 19 +++ bot/downloader/reddit/client.py | 24 +++- bot/downloader/threads/client.py | 3 + bot/downloader/tiktok/client.py | 17 +++ bot/downloader/twenty4ur/client.py | 11 ++ bot/downloader/twitch/client.py | 3 + bot/downloader/twitter/client.py | 47 ++++++- bot/downloader/youtube/client.py | 5 + bot/exceptions.py | 5 + bot/integrations/discord/client.py | 72 +++++++++++ bot/service.py | 73 +++++++++++ 16 files changed, 430 insertions(+), 121 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index e2954e5..ffdb6c1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -318,11 +318,11 @@ }, "certifi": { "hashes": [ - "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b", - "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90" + "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", + "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" ], "markers": "python_version >= '3.6'", - "version": "==2024.7.4" + "version": "==2024.8.30" }, "charset-normalizer": { "hashes": [ @@ -714,11 +714,11 @@ }, "httpx": { "hashes": [ - "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5", - "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5" + "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0", + "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2" ], "markers": "python_version >= '3.8'", - "version": "==0.27.0" + "version": "==0.27.2" }, "idna": { "hashes": [ @@ -1655,11 +1655,11 @@ }, "setuptools": { "hashes": [ - "sha256:b208925fcb9f7af924ed2dc04708ea89791e24bde0d3020b27df0e116088b34e", - "sha256:d59a3e788ab7e012ab2c4baed1b376da6366883ee20d7a5fc426816e3d7b1193" + "sha256:0274581a0037b638b9fc1c6883cc71c0210865aaa76073f7882376b641b84e8f", + "sha256:a85e96b8be2b906f3e3e789adec6a9323abf79758ecfa3065bd740d81158b11e" ], "markers": "python_version >= '3.8'", - "version": "==73.0.1" + "version": "==74.0.0" }, "six": { "hashes": [ @@ -1743,7 +1743,7 @@ "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8" ], - "markers": "python_version < '3.13'", + "markers": "python_version >= '3.8'", "version": "==4.12.2" }, "tzlocal": { @@ -1779,95 +1779,95 @@ }, "websockets": { "hashes": [ - "sha256:02cc9bb1a887dac0e08bf657c5d00aa3fac0d03215d35a599130c2034ae6663a", - "sha256:038e7a0f1bfafc7bf52915ab3506b7a03d1e06381e9f60440c856e8918138151", - "sha256:05c25f7b849702950b6fd0e233989bb73a0d2bc83faa3b7233313ca395205f6d", - "sha256:06b3186e97bf9a33921fa60734d5ed90f2a9b407cce8d23c7333a0984049ef61", - "sha256:06df8306c241c235075d2ae77367038e701e53bc8c1bb4f6644f4f53aa6dedd0", - "sha256:0a8f7d65358a25172db00c69bcc7df834155ee24229f560d035758fd6613111a", - "sha256:1f661a4205741bdc88ac9c2b2ec003c72cee97e4acd156eb733662ff004ba429", - "sha256:265e1f0d3f788ce8ef99dca591a1aec5263b26083ca0934467ad9a1d1181067c", - "sha256:2be1382a4daa61e2f3e2be3b3c86932a8db9d1f85297feb6e9df22f391f94452", - "sha256:2e1cf4e1eb84b4fd74a47688e8b0940c89a04ad9f6937afa43d468e71128cd68", - "sha256:337837ac788d955728b1ab01876d72b73da59819a3388e1c5e8e05c3999f1afa", - "sha256:358d37c5c431dd050ffb06b4b075505aae3f4f795d7fff9794e5ed96ce99b998", - "sha256:35c2221b539b360203f3f9ad168e527bf16d903e385068ae842c186efb13d0ea", - "sha256:3670def5d3dfd5af6f6e2b3b243ea8f1f72d8da1ef927322f0703f85c90d9603", - "sha256:372f46a0096cfda23c88f7e42349a33f8375e10912f712e6b496d3a9a557290f", - "sha256:376a43a4fd96725f13450d3d2e98f4f36c3525c562ab53d9a98dd2950dca9a8a", - "sha256:384129ad0490e06bab2b98c1da9b488acb35bb11e2464c728376c6f55f0d45f3", - "sha256:3a20cf14ba7b482c4a1924b5e061729afb89c890ca9ed44ac4127c6c5986e424", - "sha256:3e6566e79c8c7cbea75ec450f6e1828945fc5c9a4769ceb1c7b6e22470539712", - "sha256:4782ec789f059f888c1e8fdf94383d0e64b531cffebbf26dd55afd53ab487ca4", - "sha256:4d70c89e3d3b347a7c4d3c33f8d323f0584c9ceb69b82c2ef8a174ca84ea3d4a", - "sha256:516062a0a8ef5ecbfa4acbaec14b199fc070577834f9fe3d40800a99f92523ca", - "sha256:5575031472ca87302aeb2ce2c2349f4c6ea978c86a9d1289bc5d16058ad4c10a", - "sha256:587245f0704d0bb675f919898d7473e8827a6d578e5a122a21756ca44b811ec8", - "sha256:602cbd010d8c21c8475f1798b705bb18567eb189c533ab5ef568bc3033fdf417", - "sha256:6058b6be92743358885ad6dcdecb378fde4a4c74d4dd16a089d07580c75a0e80", - "sha256:63b702fb31e3f058f946ccdfa551f4d57a06f7729c369e8815eb18643099db37", - "sha256:6ad684cb7efce227d756bae3e8484f2e56aa128398753b54245efdfbd1108f2c", - "sha256:6fd757f313c13c34dae9f126d3ba4cf97175859c719e57c6a614b781c86b617e", - "sha256:7334752052532c156d28b8eaf3558137e115c7871ea82adff69b6d94a7bee273", - "sha256:788bc841d250beccff67a20a5a53a15657a60111ef9c0c0a97fbdd614fae0fe2", - "sha256:7d14901fdcf212804970c30ab9ee8f3f0212e620c7ea93079d6534863444fb4e", - "sha256:7ea9c9c7443a97ea4d84d3e4d42d0e8c4235834edae652993abcd2aff94affd7", - "sha256:81a11a1ddd5320429db47c04d35119c3e674d215173d87aaeb06ae80f6e9031f", - "sha256:851fd0afb3bc0b73f7c5b5858975d42769a5fdde5314f4ef2c106aec63100687", - "sha256:85a1f92a02f0b8c1bf02699731a70a8a74402bb3f82bee36e7768b19a8ed9709", - "sha256:89d795c1802d99a643bf689b277e8604c14b5af1bc0a31dade2cd7a678087212", - "sha256:9202c0010c78fad1041e1c5285232b6508d3633f92825687549540a70e9e5901", - "sha256:939a16849d71203628157a5e4a495da63967c744e1e32018e9b9e2689aca64d4", - "sha256:93b8c2008f372379fb6e5d2b3f7c9ec32f7b80316543fd3a5ace6610c5cde1b0", - "sha256:94c1c02721139fe9940b38d28fb15b4b782981d800d5f40f9966264fbf23dcc8", - "sha256:9895df6cd0bfe79d09bcd1dbdc03862846f26fbd93797153de954306620c1d00", - "sha256:9cc7f35dcb49a4e32db82a849fcc0714c4d4acc9d2273aded2d61f87d7f660b7", - "sha256:9ed02c604349068d46d87ef4c2012c112c791f2bec08671903a6bb2bd9c06784", - "sha256:a00e1e587c655749afb5b135d8d3edcfe84ec6db864201e40a882e64168610b3", - "sha256:a1ab8f0e0cadc5be5f3f9fa11a663957fecbf483d434762c8dfb8aa44948944a", - "sha256:a4de299c947a54fca9ce1c5fd4a08eb92ffce91961becb13bd9195f7c6e71b47", - "sha256:a7fbf2a8fe7556a8f4e68cb3e736884af7bf93653e79f6219f17ebb75e97d8f0", - "sha256:ad4fa707ff9e2ffee019e946257b5300a45137a58f41fbd9a4db8e684ab61528", - "sha256:ad818cdac37c0ad4c58e51cb4964eae4f18b43c4a83cb37170b0d90c31bd80cf", - "sha256:addf0a16e4983280efed272d8cb3b2e05f0051755372461e7d966b80a6554e16", - "sha256:ae7a519a56a714f64c3445cabde9fc2fc927e7eae44f413eae187cddd9e54178", - "sha256:b32f38bc81170fd56d0482d505b556e52bf9078b36819a8ba52624bd6667e39e", - "sha256:b5407c34776b9b77bd89a5f95eb0a34aaf91889e3f911c63f13035220eb50107", - "sha256:b7bf950234a482b7461afdb2ec99eee3548ec4d53f418c7990bb79c620476602", - "sha256:b89849171b590107f6724a7b0790736daead40926ddf47eadf998b4ff51d6414", - "sha256:bcea3eb58c09c3a31cc83b45c06d5907f02ddaf10920aaa6443975310f699b95", - "sha256:bd4ba86513430513e2aa25a441bb538f6f83734dc368a2c5d18afdd39097aa33", - "sha256:bf8eb5dca4f484a60f5327b044e842e0d7f7cdbf02ea6dc4a4f811259f1f1f0b", - "sha256:c026ee729c4ce55708a14b839ba35086dfae265fc12813b62d34ce33f4980c1c", - "sha256:c210d1460dc8d326ffdef9703c2f83269b7539a1690ad11ae04162bc1878d33d", - "sha256:c8feb8e19ef65c9994e652c5b0324abd657bedd0abeb946fb4f5163012c1e730", - "sha256:cbac2eb7ce0fac755fb983c9247c4a60c4019bcde4c0e4d167aeb17520cc7ef1", - "sha256:cbfe82a07596a044de78bb7a62519e71690c5812c26c5f1d4b877e64e4f46309", - "sha256:d3f3d2e20c442b58dbac593cb1e02bc02d149a86056cc4126d977ad902472e3b", - "sha256:d42a818e634f789350cd8fb413a3f5eec1cf0400a53d02062534c41519f5125c", - "sha256:d4b83cf7354cbbc058e97b3e545dceb75b8d9cf17fd5a19db419c319ddbaaf7a", - "sha256:d9726d2c9bd6aed8cb994d89b3910ca0079406edce3670886ec828a73e7bdd53", - "sha256:da7e501e59857e8e3e9d10586139dc196b80445a591451ca9998aafba1af5278", - "sha256:da7e918d82e7bdfc6f66d31febe1b2e28a1ca3387315f918de26f5e367f61572", - "sha256:dbbac01e80aee253d44c4f098ab3cc17c822518519e869b284cfbb8cd16cc9de", - "sha256:df5c0eff91f61b8205a6c9f7b255ff390cdb77b61c7b41f79ca10afcbb22b6cb", - "sha256:e07e76c49f39c5b45cbd7362b94f001ae209a3ea4905ae9a09cfd53b3c76373d", - "sha256:e1e10b3fbed7be4a59831d3a939900e50fcd34d93716e433d4193a4d0d1d335d", - "sha256:e39d393e0ab5b8bd01717cc26f2922026050188947ff54fe6a49dc489f7750b7", - "sha256:e5ba5e9b332267d0f2c33ede390061850f1ac3ee6cd1bdcf4c5ea33ead971966", - "sha256:e7a1963302947332c3039e3f66209ec73b1626f8a0191649e0713c391e9f5b0d", - "sha256:e7fcad070dcd9ad37a09d89a4cbc2a5e3e45080b88977c0da87b3090f9f55ead", - "sha256:eae368cac85adc4c7dc3b0d5f84ffcca609d658db6447387300478e44db70796", - "sha256:ede95125a30602b1691a4b1da88946bf27dae283cf30f22cd2cb8ca4b2e0d119", - "sha256:f5737c53eb2c8ed8f64b50d3dafd3c1dae739f78aa495a288421ac1b3de82717", - "sha256:f5f9d23fbbf96eefde836d9692670bfc89e2d159f456d499c5efcf6a6281c1af", - "sha256:f66e00e42f25ca7e91076366303e11c82572ca87cc5aae51e6e9c094f315ab41", - "sha256:f9af457ed593e35f467140d8b61d425495b127744a9d65d45a366f8678449a23", - "sha256:fa0839f35322f7b038d8adcf679e2698c3a483688cc92e3bd15ee4fb06669e9a", - "sha256:fd038bc9e2c134847f1e0ce3191797fad110756e690c2fdd9702ed34e7a43abb" + "sha256:00fd961943b6c10ee6f0b1130753e50ac5dcd906130dcd77b0003c3ab797d026", + "sha256:03d3f9ba172e0a53e37fa4e636b86cc60c3ab2cfee4935e66ed1d7acaa4625ad", + "sha256:0513c727fb8adffa6d9bf4a4463b2bade0186cbd8c3604ae5540fae18a90cb99", + "sha256:05e70fec7c54aad4d71eae8e8cab50525e899791fc389ec6f77b95312e4e9920", + "sha256:0617fd0b1d14309c7eab6ba5deae8a7179959861846cbc5cb528a7531c249448", + "sha256:06c0a667e466fcb56a0886d924b5f29a7f0886199102f0a0e1c60a02a3751cb4", + "sha256:0f52504023b1480d458adf496dc1c9e9811df4ba4752f0bc1f89ae92f4f07d0c", + "sha256:10a0dc7242215d794fb1918f69c6bb235f1f627aaf19e77f05336d147fce7c37", + "sha256:11f9976ecbc530248cf162e359a92f37b7b282de88d1d194f2167b5e7ad80ce3", + "sha256:132511bfd42e77d152c919147078460c88a795af16b50e42a0bd14f0ad71ddd2", + "sha256:139add0f98206cb74109faf3611b7783ceafc928529c62b389917a037d4cfdf4", + "sha256:14b9c006cac63772b31abbcd3e3abb6228233eec966bf062e89e7fa7ae0b7333", + "sha256:15c7d62ee071fa94a2fc52c2b472fed4af258d43f9030479d9c4a2de885fd543", + "sha256:165bedf13556f985a2aa064309baa01462aa79bf6112fbd068ae38993a0e1f1b", + "sha256:17118647c0ea14796364299e942c330d72acc4b248e07e639d34b75067b3cdd8", + "sha256:1841c9082a3ba4a05ea824cf6d99570a6a2d8849ef0db16e9c826acb28089e8f", + "sha256:1a678532018e435396e37422a95e3ab87f75028ac79570ad11f5bf23cd2a7d8c", + "sha256:1ee4cc030a4bdab482a37462dbf3ffb7e09334d01dd37d1063be1136a0d825fa", + "sha256:1f3cf6d6ec1142412d4535adabc6bd72a63f5f148c43fe559f06298bc21953c9", + "sha256:1f613289f4a94142f914aafad6c6c87903de78eae1e140fa769a7385fb232fdf", + "sha256:1fa082ea38d5de51dd409434edc27c0dcbd5fed2b09b9be982deb6f0508d25bc", + "sha256:249aab278810bee585cd0d4de2f08cfd67eed4fc75bde623be163798ed4db2eb", + "sha256:254ecf35572fca01a9f789a1d0f543898e222f7b69ecd7d5381d8d8047627bdb", + "sha256:2a02b0161c43cc9e0232711eff846569fad6ec836a7acab16b3cf97b2344c060", + "sha256:30d3a1f041360f029765d8704eae606781e673e8918e6b2c792e0775de51352f", + "sha256:3624fd8664f2577cf8de996db3250662e259bfbc870dd8ebdcf5d7c6ac0b5185", + "sha256:3f55b36d17ac50aa8a171b771e15fbe1561217510c8768af3d546f56c7576cdc", + "sha256:46af561eba6f9b0848b2c9d2427086cabadf14e0abdd9fde9d72d447df268418", + "sha256:47236c13be337ef36546004ce8c5580f4b1150d9538b27bf8a5ad8edf23ccfab", + "sha256:4a365bcb7be554e6e1f9f3ed64016e67e2fa03d7b027a33e436aecf194febb63", + "sha256:4d6ece65099411cfd9a48d13701d7438d9c34f479046b34c50ff60bb8834e43e", + "sha256:4e85f46ce287f5c52438bb3703d86162263afccf034a5ef13dbe4318e98d86e7", + "sha256:4f0426d51c8f0926a4879390f53c7f5a855e42d68df95fff6032c82c888b5f36", + "sha256:518f90e6dd089d34eaade01101fd8a990921c3ba18ebbe9b0165b46ebff947f0", + "sha256:52aed6ef21a0f1a2a5e310fb5c42d7555e9c5855476bbd7173c3aa3d8a0302f2", + "sha256:556e70e4f69be1082e6ef26dcb70efcd08d1850f5d6c5f4f2bcb4e397e68f01f", + "sha256:56a952fa2ae57a42ba7951e6b2605e08a24801a4931b5644dfc68939e041bc7f", + "sha256:59197afd478545b1f73367620407b0083303569c5f2d043afe5363676f2697c9", + "sha256:5df891c86fe68b2c38da55b7aea7095beca105933c697d719f3f45f4220a5e0e", + "sha256:63848cdb6fcc0bf09d4a155464c46c64ffdb5807ede4fb251da2c2692559ce75", + "sha256:64a11aae1de4c178fa653b07d90f2fb1a2ed31919a5ea2361a38760192e1858b", + "sha256:6724b554b70d6195ba19650fef5759ef11346f946c07dbbe390e039bcaa7cc3d", + "sha256:67494e95d6565bf395476e9d040037ff69c8b3fa356a886b21d8422ad86ae075", + "sha256:67648f5e50231b5a7f6d83b32f9c525e319f0ddc841be0de64f24928cd75a603", + "sha256:68264802399aed6fe9652e89761031acc734fc4c653137a5911c2bfa995d6d6d", + "sha256:699ba9dd6a926f82a277063603fc8d586b89f4cb128efc353b749b641fcddda7", + "sha256:6aa74a45d4cdc028561a7d6ab3272c8b3018e23723100b12e58be9dfa5a24491", + "sha256:6b41a1b3b561f1cba8321fb32987552a024a8f67f0d05f06fcf29f0090a1b956", + "sha256:71e6e5a3a3728886caee9ab8752e8113670936a193284be9d6ad2176a137f376", + "sha256:7d20516990d8ad557b5abeb48127b8b779b0b7e6771a265fa3e91767596d7d97", + "sha256:80e4ba642fc87fa532bac07e5ed7e19d56940b6af6a8c61d4429be48718a380f", + "sha256:872afa52a9f4c414d6955c365b6588bc4401272c629ff8321a55f44e3f62b553", + "sha256:8eb2b9a318542153674c6e377eb8cb9ca0fc011c04475110d3477862f15d29f0", + "sha256:9bbc525f4be3e51b89b2a700f5746c2a6907d2e2ef4513a8daafc98198b92237", + "sha256:a1a2e272d067030048e1fe41aa1ec8cfbbaabce733b3d634304fa2b19e5c897f", + "sha256:a5dc0c42ded1557cc7c3f0240b24129aefbad88af4f09346164349391dea8e58", + "sha256:acab3539a027a85d568c2573291e864333ec9d912675107d6efceb7e2be5d980", + "sha256:acbebec8cb3d4df6e2488fbf34702cbc37fc39ac7abf9449392cefb3305562e9", + "sha256:ad327ac80ba7ee61da85383ca8822ff808ab5ada0e4a030d66703cc025b021c4", + "sha256:b448a0690ef43db5ef31b3a0d9aea79043882b4632cfc3eaab20105edecf6097", + "sha256:b5a06d7f60bc2fc378a333978470dfc4e1415ee52f5f0fce4f7853eb10c1e9df", + "sha256:b74593e9acf18ea5469c3edaa6b27fa7ecf97b30e9dabd5a94c4c940637ab96e", + "sha256:b79915a1179a91f6c5f04ece1e592e2e8a6bd245a0e45d12fd56b2b59e559a32", + "sha256:b80f0c51681c517604152eb6a572f5a9378f877763231fddb883ba2f968e8817", + "sha256:b8ac5b46fd798bbbf2ac6620e0437c36a202b08e1f827832c4bf050da081b501", + "sha256:c3c493d0e5141ec055a7d6809a28ac2b88d5b878bb22df8c621ebe79a61123d0", + "sha256:c44ca9ade59b2e376612df34e837013e2b273e6c92d7ed6636d0556b6f4db93d", + "sha256:c4a6343e3b0714e80da0b0893543bf9a5b5fa71b846ae640e56e9abc6fbc4c83", + "sha256:c5870b4a11b77e4caa3937142b650fbbc0914a3e07a0cf3131f35c0587489c1c", + "sha256:ca48914cdd9f2ccd94deab5bcb5ac98025a5ddce98881e5cce762854a5de330b", + "sha256:cf2fae6d85e5dc384bf846f8243ddaa9197f3a1a70044f59399af001fd1f51d4", + "sha256:d450f5a7a35662a9b91a64aefa852f0c0308ee256122f5218a42f1d13577d71e", + "sha256:d6716c087e4aa0b9260c4e579bb82e068f84faddb9bfba9906cb87726fa2e870", + "sha256:d93572720d781331fb10d3da9ca1067817d84ad1e7c31466e9f5e59965618096", + "sha256:dbb0b697cc0655719522406c059eae233abaa3243821cfdfab1215d02ac10231", + "sha256:e33505534f3f673270dd67f81e73550b11de5b538c56fe04435d63c02c3f26b5", + "sha256:e801ca2f448850685417d723ec70298feff3ce4ff687c6f20922c7474b4746ae", + "sha256:e82db3756ccb66266504f5a3de05ac6b32f287faacff72462612120074103329", + "sha256:ef48e4137e8799998a343706531e656fdec6797b80efd029117edacb74b0a10a", + "sha256:f1d3d1f2eb79fe7b0fb02e599b2bf76a7619c79300fc55f0b5e2d382881d4f7f", + "sha256:f3fea72e4e6edb983908f0db373ae0732b275628901d909c382aae3b592589f2", + "sha256:f40de079779acbcdbb6ed4c65af9f018f8b77c5ec4e17a4b737c05c2db554491", + "sha256:f73e676a46b0fe9426612ce8caeca54c9073191a77c3e9d5c94697aef99296af", + "sha256:f9c9e258e3d5efe199ec23903f5da0eeaad58cf6fccb3547b74fd4750e5ac47a", + "sha256:fac2d146ff30d9dd2fcf917e5d147db037a5c573f0446c564f16f1f94cf87462", + "sha256:faef9ec6354fe4f9a2c0bbb52fb1ff852effc897e2a4501e25eb3a47cb0a4f89" ], "markers": "python_version >= '3.8'", - "version": "==13.0" + "version": "==13.0.1" }, "wells": { "hashes": [ @@ -2144,11 +2144,11 @@ }, "setuptools": { "hashes": [ - "sha256:b208925fcb9f7af924ed2dc04708ea89791e24bde0d3020b27df0e116088b34e", - "sha256:d59a3e788ab7e012ab2c4baed1b376da6366883ee20d7a5fc426816e3d7b1193" + "sha256:0274581a0037b638b9fc1c6883cc71c0210865aaa76073f7882376b641b84e8f", + "sha256:a85e96b8be2b906f3e3e789adec6a9323abf79758ecfa3065bd740d81158b11e" ], "markers": "python_version >= '3.8'", - "version": "==73.0.1" + "version": "==74.0.0" }, "tomlkit": { "hashes": [ diff --git a/bot/common/utils.py b/bot/common/utils.py index d2df73f..e3ba56b 100644 --- a/bot/common/utils.py +++ b/bot/common/utils.py @@ -1,4 +1,5 @@ import asyncio +import datetime import io import mimetypes import os @@ -110,3 +111,19 @@ def temp_open(path: str, mode: str = 'rb'): finally: f.close() os.remove(path) + + +def number_to_human_format(number: int) -> str: + num = float('{:.3g}'.format(number)) + magnitude = 0 + while abs(num) >= 1000: + magnitude += 1 + num /= 1000.0 + return '{}{}'.format('{:f}'.format(num).rstrip('0').rstrip('.'), ['', 'K', 'M', 'B', 'T'][magnitude]) + + +def date_to_human_format(date: datetime.datetime) -> str: + if date.hour == 0 and date.minute == 0: + return date.strftime('%b %-d, %Y') + + return date.strftime('%H:%M ยท %b %-d, %Y') diff --git a/bot/domain.py b/bot/domain.py index d735cec..e625fb2 100644 --- a/bot/domain.py +++ b/bot/domain.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from bot import constants +from bot.common import utils DEFAULT_POST_FORMAT = """๐Ÿ”— URL: {url} @@ -14,6 +15,12 @@ ๐Ÿ“• Description: {description}\n """ +DEFAULT_COMMENT_FORMAT = """๐Ÿง‘๐Ÿปโ€๐ŸŽจ Author: {author} +๐Ÿ“… Created: {created} +๐Ÿ‘๐Ÿป Likes: {likes} +๐Ÿ“• Comment: {comment}\n +""" + SERVER_INFO_FORMAT = """```yml Tier: {tier} Prefix: {prefix} @@ -100,10 +107,10 @@ def __str__(self) -> str: return self._format.format( url=self.url, author=self.author or 'โŒ', - created=self._date_human_format(date=self.created) if self.created else 'โŒ', + created=utils.date_to_human_format(self.created) if self.created else 'โŒ', description=description if not self.spoiler else f'||{description}||', - views=self._number_human_format(num=self.views) if self.views else 'โŒ', - likes=self._number_human_format(num=self.likes) if self.likes else 'โŒ', + views=utils.number_to_human_format(self.views) if self.views else 'โŒ', + likes=utils.number_to_human_format(self.likes) if self.likes else 'โŒ', ) def set_format(self, fmt: typing.Optional[str]) -> None: @@ -118,16 +125,29 @@ def read_buffer(self) -> typing.Optional[bytes]: self.buffer.seek(0) return res - def _number_human_format(self, num: int) -> str: - num = float('{:.3g}'.format(num)) - magnitude = 0 - while abs(num) >= 1000: - magnitude += 1 - num /= 1000.0 - return '{}{}'.format('{:f}'.format(num).rstrip('0').rstrip('.'), ['', 'K', 'M', 'B', 'T'][magnitude]) - def _date_human_format(self, date: datetime.datetime) -> str: - if date.hour == 0 and date.minute == 0: - return date.strftime('%b %-d, %Y') +@dataclass +class Comment: + author: typing.Optional[str] = None + created: typing.Optional[datetime.datetime] = None + likes: typing.Optional[int] = None + comment: typing.Optional[str] = None + spoiler: bool = False + _format: str = DEFAULT_COMMENT_FORMAT + + def __str__(self) -> str: + comment = self.comment or 'โŒ' + + return self._format.format( + author=self.author or 'โŒ', + created=utils.date_to_human_format(self.created) if self.created else 'โŒ', + likes=utils.number_to_human_format(self.likes) if self.likes else 'โŒ', + comment=comment if not self.spoiler else f'||{comment}||', + ) + + def set_format(self, fmt: typing.Optional[str]) -> None: + self._format = fmt or DEFAULT_COMMENT_FORMAT + - return date.strftime('%H:%M ยท %b %-d, %Y') +def comments_to_string(comments: typing.List[Comment]) -> str: + return ''.join([str(comment) for comment in comments]) diff --git a/bot/downloader/base.py b/bot/downloader/base.py index 43871d6..d7ec3e3 100644 --- a/bot/downloader/base.py +++ b/bot/downloader/base.py @@ -20,6 +20,9 @@ async def get_integration_data(self, url: str) -> typing.Tuple[constants.Integra async def get_post(self, url: str) -> domain.Post: raise NotImplementedError() + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + raise NotImplementedError() + async def _download(self, url: str, cookies: typing.Optional[typing.Dict[str, str]] = None, **kwargs) -> io.BytesIO: async with aiohttp.ClientSession(cookies=cookies) as session: async with session.get(url=url, **kwargs) as resp: diff --git a/bot/downloader/facebook/client.py b/bot/downloader/facebook/client.py index 5a6172e..6c6a318 100644 --- a/bot/downloader/facebook/client.py +++ b/bot/downloader/facebook/client.py @@ -7,6 +7,7 @@ from bot import constants from bot import domain +from bot import exceptions from bot import logger from bot.downloader import base from bot.downloader.facebook import config @@ -68,3 +69,6 @@ async def get_post(self, url: str) -> domain.Post: post.buffer = await self._download(url=fb_post['images'][0]) return post + + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + raise exceptions.NotSupportedError('get_comments') diff --git a/bot/downloader/instagram/client.py b/bot/downloader/instagram/client.py index c89e250..fe74c59 100644 --- a/bot/downloader/instagram/client.py +++ b/bot/downloader/instagram/client.py @@ -70,6 +70,25 @@ async def get_post(self, url: str) -> domain.Post: raise NotImplementedError(f'Not yet implemented for {url}') + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + uid, _, _ = self._parse_url(url) + p = instaloader.Post.from_shortcode(context=self.client.context, shortcode=uid) + + comments = [] + for i, comment in enumerate(p.get_comments()): + comments.append( + domain.Comment( + author=comment.owner.username, + created=comment.created_at_utc, + likes=comment.likes_count, + comment=comment.text, + ) + ) + if i + 1 == n: + break + + return comments + @staticmethod def _parse_url(url: str) -> typing.Tuple[str, int, constants.LinkType]: parsed_url = urlparse(url) diff --git a/bot/downloader/reddit/client.py b/bot/downloader/reddit/client.py index 0c20c64..26855f7 100644 --- a/bot/downloader/reddit/client.py +++ b/bot/downloader/reddit/client.py @@ -1,6 +1,6 @@ import datetime -import io import glob +import io import os import re import shutil @@ -80,6 +80,28 @@ async def get_post(self, url: str) -> domain.Post: return post + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + if not self.client: + raise exceptions.ConfigurationError('Reddit credentials not configured') + + try: + submission = await self.client.submission(url=url) + except praw_exceptions.InvalidURL: + # Hack for new reddit urls generated in mobile app + # Does another request, which redirects to the correct url + url = requests.get(url, timeout=base.DEFAULT_TIMEOUT).url.split('?')[0] + submission = await self.client.submission(url=url) + + return [ + domain.Comment( + author=comment.author, + created=datetime.datetime.fromtimestamp(comment.created_utc).astimezone(), + likes=comment.score, + comment=comment.body, + ) + for comment in submission.comments[:n] + ] + async def _hydrate_post(self, post: domain.Post) -> bool: if not self.client: return self._hydrate_post_no_login(post) diff --git a/bot/downloader/threads/client.py b/bot/downloader/threads/client.py index 49f1361..f7cacd4 100644 --- a/bot/downloader/threads/client.py +++ b/bot/downloader/threads/client.py @@ -97,6 +97,9 @@ async def get_post(self, url: str) -> domain.Post: return post + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + raise exceptions.NotSupportedError('get_comments') + def _get_thread_id(self, url_id: str) -> str: alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_' diff --git a/bot/downloader/tiktok/client.py b/bot/downloader/tiktok/client.py index 4f67da1..31e9acc 100644 --- a/bot/downloader/tiktok/client.py +++ b/bot/downloader/tiktok/client.py @@ -76,6 +76,23 @@ async def get_post(self, url: str) -> domain.Post: created=video.create_time.astimezone(), ) + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + clean_url = self._clean_url(url) + + async with AsyncTikTokAPI() as api: + video = await api.video(clean_url) + + logger.debug('Trying to fetch tiktok comments', url=url) + + return [ + domain.Comment( + author=comment.user.unique_id if isinstance(comment.user, user.LightUser) else comment.user, + likes=comment.digg_count, + comment=comment.text, + ) + async for comment in video.comments.limit(n) + ] + async def _download_slideshow(self, video: tiktok_video.Video, cookies: typing.Dict[str, str]) -> io.BytesIO: vf = ( '"scale=iw*min(1080/iw\\,1920/ih):ih*min(1080/iw\\,1920/ih),' diff --git a/bot/downloader/twenty4ur/client.py b/bot/downloader/twenty4ur/client.py index ef66011..e047839 100644 --- a/bot/downloader/twenty4ur/client.py +++ b/bot/downloader/twenty4ur/client.py @@ -54,3 +54,14 @@ async def get_post(self, url: str) -> domain.Post: views=article.num_views, buffer=buffer, ) + + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + return [ + domain.Comment( + author=comment.author, + created=comment.posted_at, + likes=comment.score, + comment=comment.content, + ) + for comment in (await self.client.get_article_by_url(url=url, num_comments=n)).comments + ] diff --git a/bot/downloader/twitch/client.py b/bot/downloader/twitch/client.py index c59ca0a..8ea64e8 100644 --- a/bot/downloader/twitch/client.py +++ b/bot/downloader/twitch/client.py @@ -58,6 +58,9 @@ async def get_post(self, url: str) -> domain.Post: buffer=io.BytesIO(resp.content), ) + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + raise exceptions.NotSupportedError('get_comments') + @staticmethod def _find_quality(qualities: typing.List[twitch.VideoQuality], max_quality: int = 720) -> int: """ diff --git a/bot/downloader/twitter/client.py b/bot/downloader/twitter/client.py index e5e1c10..671c4ea 100644 --- a/bot/downloader/twitter/client.py +++ b/bot/downloader/twitter/client.py @@ -84,12 +84,7 @@ async def relogin(self) -> None: if self.client: await self.client.pool.relogin(usernames=[self.username]) - async def get_post(self, url: str) -> domain.Post: - uid, index = self._parse_url(url) - - if self.client is None: - return await self._get_post_no_login(url=url, uid=uid, index=index or 0) - + async def login(self) -> None: if not self.logged_in: await self.client.pool.add_account( username=self.username, @@ -99,8 +94,48 @@ async def get_post(self, url: str) -> domain.Post: ) await self.client.pool.login_all() + async def get_post(self, url: str) -> domain.Post: + uid, index = self._parse_url(url) + + if self.client is None: + return await self._get_post_no_login(url=url, uid=uid, index=index or 0) + + await self.login() + return await self._get_post_login(url=url, uid=uid, index=index) + async def get_comments( + self, + url: str, + n: int = 5, + retry_count: int = 0, + ) -> typing.List[domain.Comment]: + if not self.client: + raise exceptions.NotAllowedError('Twitter credentials not configured') + + await self.login() + + uid, _ = self._parse_url(url) + try: + replies = self.client.tweet_replies(twid=int(uid), limit=n) + except Exception as e: + logger.error('Failed fetching from twitter, retrying', error=str(e)) + if retry_count == 0: + await self.relogin() + return await self.get_comments(url=url, n=n, retry_count=retry_count + 1) + + raise exceptions.IntegrationClientError('Failed fetching from twitter') from e + + return [ + domain.Comment( + author=f'{reply.user.displayname} ({reply.user.username})', + created=reply.date.astimezone(), + likes=reply.likeCount, + comment=reply.rawContent, + ) + async for reply in replies + ][:n] + async def _get_post_login( self, url: str, diff --git a/bot/downloader/youtube/client.py b/bot/downloader/youtube/client.py index 5bb826e..edff499 100644 --- a/bot/downloader/youtube/client.py +++ b/bot/downloader/youtube/client.py @@ -6,6 +6,7 @@ from bot import constants from bot import domain +from bot import exceptions from bot import logger from bot.downloader import base from bot.downloader.youtube import config @@ -43,6 +44,7 @@ async def get_post(self, url: str) -> domain.Post: views=vid.views, created=vid.publish_date, buffer=io.BytesIO(), + spoiler=vid.age_restricted is True, ) vid.streams.filter(progressive=True, file_extension='mp4').order_by( @@ -50,3 +52,6 @@ async def get_post(self, url: str) -> domain.Post: ).desc().first().stream_to_buffer(post.buffer) return post + + async def get_comments(self, url: str, n: int = 5) -> typing.List[domain.Comment]: + raise exceptions.NotSupportedError('get_comments') diff --git a/bot/exceptions.py b/bot/exceptions.py index 7cf7edf..a83daf3 100644 --- a/bot/exceptions.py +++ b/bot/exceptions.py @@ -17,6 +17,11 @@ def __init__(self, action: str) -> None: super().__init__(f'Action not allowed: {action}') +class NotSupportedError(BaseError): + def __init__(self, action: str) -> None: + super().__init__(f'Action not supported: {action}') + + class ConfigurationError(BaseError): pass diff --git a/bot/integrations/discord/client.py b/bot/integrations/discord/client.py index 3291ec2..3152f70 100644 --- a/bot/integrations/discord/client.py +++ b/bot/integrations/discord/client.py @@ -2,6 +2,7 @@ import datetime import typing from functools import partial +from itertools import batched import discord from discord import app_commands @@ -51,6 +52,11 @@ def __init__(self, *, intents: discord.Intents, **options: typing.Any) -> None: description='Embeds media directly into discord', callback=self.embed_cmd, ), + app_commands.Command( + name='comments', + description='Fetches comments for a post', + callback=self.get_comments_cmd, + ), app_commands.Command( name='help', description='Prints configuration for this server', @@ -181,6 +187,47 @@ async def embed_cmd(self, interaction: discord.Interaction, url: str, spoiler: b author=interaction.user, ) + async def get_comments_cmd( + self, + interaction: discord.Interaction, + url: str, + n: int = 5, + spoiler: bool = False, + ) -> None: + await interaction.response.defer() + + if service.should_handle_url(url) is False: + return + + try: + comments = await service.get_comments( + url=url, + n=n, + server_vendor=constants.ServerVendor.DISCORD, + server_uid=str(interaction.guild_id), + author_uid=str(interaction.user.id), + ) + except Exception as e: + logger.error('Failed downloading', url=url, error=str(e)) + await interaction.followup.send( + content=f'Failed fetching {url} ({interaction.user.mention}).\nError: {str(e)}', + view=CustomView(), + ) + raise e + + # Override spoiler + for comment in comments: + if not comment.spoiler: + comment.spoiler = spoiler + + for batch in batched(comments, 5): + await self._send_comments( + url=url, + comments=batch, + send_func=partial(interaction.followup.send, view=CustomView()), + author=interaction.user, + ) + async def help_cmd(self, interaction: discord.Interaction) -> None: await interaction.response.defer() @@ -308,3 +355,28 @@ async def _send_post( return await self._send_post(post=post, send_func=send_func, author=author) raise exceptions.BotError('Failed to send message') from e + + async def _send_comments( + self, + url: str, + comments: typing.List[domain.Comment], + send_func: typing.Callable, + author: typing.Union[discord.User, discord.Member], + ) -> discord.Message: + send_kwargs = { + 'suppress_embeds': True, + } + + content = f'Here you go {author.mention} {utils.random_emoji()}.\n{url}\n{domain.comments_to_string(comments)}' + if len(content) > 2000: + if any(comment.spoiler is True for comment in comments): + content = content[:1995] + '||...' + else: + content = content[:1997] + '...' + + send_kwargs['content'] = content + + try: + return await send_func(**send_kwargs) + except discord.HTTPException as e: + raise exceptions.BotError('Failed to send message') from e diff --git a/bot/service.py b/bot/service.py index bbab849..e3185eb 100644 --- a/bot/service.py +++ b/bot/service.py @@ -141,3 +141,76 @@ async def get_post( # noqa: C901 ) return post + + +async def get_comments( # noqa: C901 + url: str, + n: int, + server_vendor: constants.ServerVendor, + server_uid: str, + author_uid: str, +) -> typing.List[domain.Comment]: + # TODO: Refactor + if n > 15: + raise exceptions.NotAllowedError('Can\'t fetch more than 15 comments') + + try: + client = registry.get_instance(url) + except ValueError as e: + logger.warning('No strategy for url', url=url, error=str(e)) + return None + + if not client: + logger.warning('Integration for url not enabled or client init failure', url=url) + return None + + # Check if server is throttled and allowed to post + server = repository.get_server( + vendor=server_vendor, + vendor_uid=server_uid, + ) + if not server: + logger.info( + 'Server not configured, creating a default config', + server_vendor_uid=server_uid, + server_vendor=server_vendor.value, + ) + server = repository.create_server(vendor=server_vendor, vendor_uid=server_uid) + + if not server._internal_id: + logger.error('Internal id for server not set') + raise exceptions.BotError('Internal server error') + + num_posts_in_server = repository.get_number_of_posts_in_server_from_datetime( + server_id=server._internal_id, + from_datetime=datetime.datetime.now() - datetime.timedelta(days=1), + ) + + if not server.can_post(num_posts_in_one_day=num_posts_in_server, integration=client.INTEGRATION): + logger.warning( + 'Server is not allowed to post', + server_vendor=server_vendor.value, + server_vendor_uid=server_uid, + server_tier=server.tier.name, + ) + raise exceptions.NotAllowedError('Upgrade your tier') + + # Check if user is banned + if repository.is_member_banned_from_server( + server_vendor=server_vendor, + server_uid=server_uid, + member_uid=author_uid, + ): + logger.warning( + 'User banned from server', + user=author_uid, + server_vendor=server_vendor.value, + server_vendor_uid=server_uid, + ) + raise exceptions.NotAllowedError('User banned') + + try: + return await client.get_comments(url=url, n=n) + except Exception as e: + logger.error('Failed downloading', url=url, num_comments=n, error=str(e)) + raise e From 101342c10913656d0118a35f6eccf9fa832b58c4 Mon Sep 17 00:00:00 2001 From: Amadej Kastelic Date: Fri, 30 Aug 2024 11:38:23 +0200 Subject: [PATCH 2/2] Bump 24ur-api, support more 24ur sub-sites --- Pipfile | 2 +- Pipfile.lock | 19 ++++++++++--------- bot/downloader/twenty4ur/client.py | 11 ++++++++++- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/Pipfile b/Pipfile index f887e18..7641ec7 100644 --- a/Pipfile +++ b/Pipfile @@ -27,7 +27,7 @@ pymemcache = "==4.0.0" "discord-oauth2.py" = "==1.2.1" twitch-dl = "==2.3.1" pydantic = "==2.8.2" -"24ur-api[download]" = "==0.1.4" +"24ur-api[video]" = "==0.1.5" [dev-packages] black = "==24.8.0" diff --git a/Pipfile.lock b/Pipfile.lock index ffdb6c1..5fa2a5f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5f554dda5077e7df95761b86a0112977633635e10d719cd35974b9aa0fd83a97" + "sha256": "06e68deac28dbd90e12cd46668596d6249d670d06099796ba4f96a346cfc6d9b" }, "pipfile-spec": 6, "requires": { @@ -18,14 +18,15 @@ "default": { "24ur-api": { "extras": [ - "download" + "download", + "video" ], "hashes": [ - "sha256:5d489d7ba4337d17ffc30305bce7edce96f76019efc3eefb12a9e6717aedb4a1", - "sha256:d4900befb9937dc03fd46f6bc8deeeb804ef87e6bfe9db609379353d1b5290f0" + "sha256:18968818cc06d6c99e3cf70c107f902e5bf725b05de4a517320893cc7b2aff1c", + "sha256:8ee54fb5d8d56d1be6ad688bc4fa6d30a3eec5cd46d3e6c0bafd8fe512b83f13" ], "markers": "python_version ~= '3.8'", - "version": "==0.1.4" + "version": "==0.1.5" }, "aiofiles": { "hashes": [ @@ -1480,10 +1481,10 @@ }, "pyquery": { "hashes": [ - "sha256:8dfc9b4b7c5f877d619bbae74b1898d5743f6ca248cfd5d72b504dd614da312f", - "sha256:963e8d4e90262ff6d8dec072ea97285dc374a2f69cad7776f4082abcf6a1d8ae" + "sha256:0194bb2706b12d037db12c51928fe9ebb36b72d9e719565daba5a6c595322faf", + "sha256:aedfa0bd0eb9afc94b3ddbec8f375a6362b32bc9662f46e3e0d866483f4771b0" ], - "version": "==2.0.0" + "version": "==2.0.1" }, "python-dateutil": { "hashes": [ @@ -1743,7 +1744,7 @@ "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8" ], - "markers": "python_version >= '3.8'", + "markers": "python_version < '3.13'", "version": "==4.12.2" }, "tzlocal": { diff --git a/bot/downloader/twenty4ur/client.py b/bot/downloader/twenty4ur/client.py index e047839..2f3af30 100644 --- a/bot/downloader/twenty4ur/client.py +++ b/bot/downloader/twenty4ur/client.py @@ -12,7 +12,16 @@ class Twenty4UrClientSingleton(base.BaseClientSingleton): - DOMAINS = ['24ur.com'] + DOMAINS = [ + '24ur.com', + 'zadovoljna.si', + 'bibaleze.si', + 'vizita.si', + 'cekin.si', + 'moskisvet.com', + 'dominvrt.si', + 'okusno.je', + ] _CONFIG_SCHEMA = config.Twenty4UrConfig @classmethod