Extract_Images.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "984526c6-ea04-4e7e-92d6-61e145df4ec7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "无法下载或保存图片 https://pbs.twimg.com/media/GHOJD4daYAAOyFh?format=jpg&name=medium: ('Connection aborted.', ConnectionResetError(54, 'Connection reset by peer'))\n",
      "下载完成。跳过重复x 534 \n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "import requests\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "\n",
    "def download_image(image_url, save_dir, mime_types, skip_count):\n",
    "    try:\n",
    "        # 提取URL中的文件名，并尝试获取文件的MIME类型（此处在函数外获取MIME类型可能更高效）\n",
    "        content_type = requests.head(image_url).headers.get('Content-Type', '')\n",
    "        extension = mime_types.get(content_type, '')\n",
    "        image_name = os.path.basename(image_url).split('?')[0] + extension\n",
    "        image_path = os.path.join(save_dir, image_name)\n",
    "\n",
    "        # 检查图片是否已存在\n",
    "        if os.path.exists(image_path):\n",
    "            skip_count['count'] += 1\n",
    "            return\n",
    "\n",
    "        # 从URL下载图片\n",
    "        response = requests.get(image_url)\n",
    "        if response.status_code == 200:\n",
    "            # 保存图片到文件\n",
    "            with open(image_path, 'wb') as image_file:\n",
    "                image_file.write(response.content)\n",
    "    except Exception as e:\n",
    "        print(f\"无法下载或保存图片 {image_url}: {e}\")\n",
    "\n",
    "# 初始化跳过下载的图片计数\n",
    "skip_count = {'count': 0}\n",
    "\n",
    "# 文件路径和目录定义\n",
    "file_path = 'data/tweets_2024-03-07_22-02-24.json'\n",
    "save_dir = '𝕏_likes'\n",
    "os.makedirs(save_dir, exist_ok=True)\n",
    "\n",
    "# MIME类型映射\n",
    "mime_types = {\n",
    "    \"image/jpeg\": \".jpg\",\n",
    "    \"image/png\": \".png\",\n",
    "    \"image/gif\": \".jpeg\"\n",
    "}\n",
    "\n",
    "# 读取并处理JSON文件\n",
    "with open(file_path, 'r', encoding='utf-8') as file:\n",
    "    tweets = [json.loads(line) for line in file.readlines()]\n",
    "\n",
    "# 使用ThreadPoolExecutor来并行处理下载任务\n",
    "with ThreadPoolExecutor(max_workers= 5) as executor:\n",
    "    future_to_url = {executor.submit(download_image, tweet['images_urls'][0], save_dir, mime_types, skip_count): tweet for tweet in tweets if tweet.get('media_type') == 'Image' and tweet.get('images_urls')}\n",
    "    for future in as_completed(future_to_url):\n",
    "        url = future_to_url[future]\n",
    "        try:\n",
    "            data = future.result()\n",
    "        except Exception as exc:\n",
    "            print(f'{url} generated an exception: {exc}')\n",
    "\n",
    "print(f\"下载完成。跳过重复x {skip_count['count']} \")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}