From ee3f9aaa8be5f745c40bc3bb5aeed6fcc1b332eb Mon Sep 17 00:00:00 2001 From: naf Date: Mon, 5 Feb 2018 01:57:19 -0600 Subject: [PATCH 1/7] webseeds initial commit --- src/download/download_constructor.cc | 17 ++ src/download/download_constructor.h | 1 + src/download/download_main.cc | 3 + src/download/download_main.h | 5 + src/torrent/Makefile.am | 7 +- src/torrent/data/file_list.h | 1 + src/torrent/download.cc | 5 + src/torrent/tracker.h | 1 + src/torrent/tracker_list.cc | 4 + src/torrent/webseed_controller.cc | 234 +++++++++++++++++++++++++++ src/torrent/webseed_controller.h | 78 +++++++++ src/tracker/Makefile.am | 4 +- src/tracker/tracker_webseed.cc | 109 +++++++++++++ src/tracker/tracker_webseed.h | 75 +++++++++ 14 files changed, 541 insertions(+), 3 deletions(-) create mode 100644 src/torrent/webseed_controller.cc create mode 100644 src/torrent/webseed_controller.h create mode 100644 src/tracker/tracker_webseed.cc create mode 100644 src/tracker/tracker_webseed.h diff --git a/src/download/download_constructor.cc b/src/download/download_constructor.cc index 1bb261bde..61d377b33 100644 --- a/src/download/download_constructor.cc +++ b/src/download/download_constructor.cc @@ -48,6 +48,7 @@ #include "torrent/object.h" #include "torrent/tracker_controller.h" #include "torrent/tracker_list.h" +#include "torrent/webseed_controller.h" #include "torrent/data/file.h" #include "torrent/data/file_list.h" @@ -209,6 +210,11 @@ DownloadConstructor::parse_tracker(const Object& b) { std::for_each(b.get_key_list("nodes").begin(), b.get_key_list("nodes").end(), rak::make_mem_fun(this, &DownloadConstructor::add_dht_node)); + if (b.has_key_list("url-list")) + { + std::for_each(b.get_key_list("url-list").begin(), b.get_key_list("url-list").end(), rak::make_mem_fun(this, &DownloadConstructor::add_webseed_url)); + } + m_download->main()->tracker_list()->randomize_group_entries(); } @@ -248,6 +254,17 @@ DownloadConstructor::add_dht_node(const Object& b) { manager->dht_manager()->add_node(host, el->as_value()); } +void +DownloadConstructor::add_webseed_url(const Object& b) { + if (!b.is_string()) + return; + + std::string url = b.as_string(); + m_download->main()->webseed_controller()->add_url(url); + m_download->main()->tracker_list()->insert_url(m_download->main()->tracker_list()->size_group(), "webseed:" + url); +} + + bool DownloadConstructor::is_valid_path_element(const Object& b) { return diff --git a/src/download/download_constructor.h b/src/download/download_constructor.h index 7d43aba38..c05607893 100644 --- a/src/download/download_constructor.h +++ b/src/download/download_constructor.h @@ -69,6 +69,7 @@ class DownloadConstructor { void add_tracker_group(const Object& b); void add_tracker_single(const Object& b, int group); void add_dht_node(const Object& b); + void add_webseed_url(const Object& b); static bool is_valid_path_element(const Object& b); static bool is_invalid_path_element(const Object& b) { return !is_valid_path_element(b); } diff --git a/src/download/download_main.cc b/src/download/download_main.cc index 48222c383..8342ea773 100644 --- a/src/download/download_main.cc +++ b/src/download/download_main.cc @@ -57,6 +57,7 @@ #include "torrent/peer/peer_info.h" #include "torrent/tracker_controller.h" #include "torrent/tracker_list.h" +#include "torrent/webseed_controller.h" #include "torrent/utils/log.h" #include "available_list.h" @@ -105,6 +106,8 @@ DownloadMain::DownloadMain() : m_tracker_list = new TrackerList(); m_tracker_controller = new TrackerController(m_tracker_list); + m_webseed_controller = new WebseedController(this); + m_tracker_list->slot_success() = std::bind(&TrackerController::receive_success, m_tracker_controller, std::placeholders::_1, std::placeholders::_2); m_tracker_list->slot_failure() = std::bind(&TrackerController::receive_failure, m_tracker_controller, std::placeholders::_1, std::placeholders::_2); m_tracker_list->slot_scrape_success() = std::bind(&TrackerController::receive_scrape, m_tracker_controller, std::placeholders::_1); diff --git a/src/download/download_main.h b/src/download/download_main.h index da3cf182f..7987f0c5c 100644 --- a/src/download/download_main.h +++ b/src/download/download_main.h @@ -64,6 +64,7 @@ class DownloadWrapper; class HandshakeManager; class TrackerController; class TrackerList; +class WebseedController; class DownloadInfo; class ThrottleList; class InitialSeeding; @@ -89,6 +90,8 @@ class DownloadMain { TrackerController* tracker_controller() { return m_tracker_controller; } TrackerList* tracker_list() { return m_tracker_list; } + WebseedController* webseed_controller() { return m_webseed_controller; } + DownloadInfo* info() { return m_info; } // Only retrieve writable chunks when the download is active. @@ -174,6 +177,8 @@ class DownloadMain { TrackerController* m_tracker_controller; TrackerList* m_tracker_list; + WebseedController* m_webseed_controller; + class choke_group* m_choke_group; group_entry m_up_group_entry; diff --git a/src/torrent/Makefile.am b/src/torrent/Makefile.am index 1bdfde3de..bfbd9568f 100644 --- a/src/torrent/Makefile.am +++ b/src/torrent/Makefile.am @@ -56,7 +56,9 @@ libsub_torrent_la_SOURCES = \ tracker_controller.cc \ tracker_controller.h \ tracker_list.cc \ - tracker_list.h + tracker_list.h \ + webseed_controller.cc \ + webseed_controller.h AM_CPPFLAGS = -I$(srcdir) -I$(srcdir)/.. -I$(top_srcdir) @@ -88,4 +90,5 @@ libtorrentinclude_HEADERS = \ torrent.h \ tracker.h \ tracker_controller.h \ - tracker_list.h + tracker_list.h \ + webseed_controller.h diff --git a/src/torrent/data/file_list.h b/src/torrent/data/file_list.h index 2cb64f669..722392633 100644 --- a/src/torrent/data/file_list.h +++ b/src/torrent/data/file_list.h @@ -62,6 +62,7 @@ class LIBTORRENT_EXPORT FileList : private std::vector { friend class DownloadMain; friend class DownloadWrapper; friend class Handshake; + friend class WebseedController; typedef std::vector base_type; typedef std::vector path_list; diff --git a/src/torrent/download.cc b/src/torrent/download.cc index edddedfbf..3c87c861f 100644 --- a/src/torrent/download.cc +++ b/src/torrent/download.cc @@ -58,6 +58,7 @@ #include "torrent/data/file.h" #include "torrent/peer/connection_list.h" #include "torrent/tracker_controller.h" +#include "torrent/webseed_controller.h" #include "torrent/tracker_list.h" #include "torrent/utils/log.h" @@ -171,6 +172,8 @@ Download::start(int flags) { if (!(flags & start_skip_tracker)) m_ptr->main()->tracker_controller()->send_start_event(); + + m_ptr->main()->webseed_controller()->start(); } void @@ -186,6 +189,8 @@ Download::stop(int flags) { m_ptr->main()->tracker_controller()->send_stop_event(); m_ptr->main()->tracker_controller()->disable(); + + m_ptr->main()->webseed_controller()->stop(); } bool diff --git a/src/torrent/tracker.h b/src/torrent/tracker.h index 2b00ad471..7c55341ab 100644 --- a/src/torrent/tracker.h +++ b/src/torrent/tracker.h @@ -55,6 +55,7 @@ class LIBTORRENT_EXPORT Tracker { TRACKER_HTTP, TRACKER_UDP, TRACKER_DHT, + TRACKER_WEBSEED } Type; enum tracker_event { diff --git a/src/torrent/tracker_list.cc b/src/torrent/tracker_list.cc index 625055da7..fdf656086 100644 --- a/src/torrent/tracker_list.cc +++ b/src/torrent/tracker_list.cc @@ -46,6 +46,7 @@ #include "tracker/tracker_dht.h" #include "tracker/tracker_http.h" #include "tracker/tracker_udp.h" +#include "tracker/tracker_webseed.h" #include "globals.h" #include "exceptions.h" @@ -203,6 +204,9 @@ TrackerList::insert_url(unsigned int group, const std::string& url, bool extra_t } else if (std::strncmp("dht://", url.c_str(), 6) == 0 && TrackerDht::is_allowed()) { tracker = new TrackerDht(this, url, flags); + } else if (std::strncmp("webseed:", url.c_str(), 8) == 0) { + tracker = new TrackerWebseed(this, url, flags); + } else { LT_LOG_TRACKER(WARN, "could find matching tracker protocol (url:%s)", url.c_str()); diff --git a/src/torrent/webseed_controller.cc b/src/torrent/webseed_controller.cc new file mode 100644 index 000000000..73411c21a --- /dev/null +++ b/src/torrent/webseed_controller.cc @@ -0,0 +1,234 @@ +// libTorrent - BitTorrent library +// Copyright (C) 2005-2011, Jari Sundell +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations +// including the two. +// +// You must obey the GNU General Public License in all respects for +// all of the code used other than OpenSSL. If you modify file(s) +// with this exception, you may extend this exception to your version +// of the file(s), but you are not obligated to do so. If you do not +// wish to do so, delete this exception statement from your version. +// If you delete this exception statement from all source files in the +// program, then also delete it here. +// +// Contact: Jari Sundell +// +// Skomakerveien 33 +// 3185 Skoppum, NORWAY + +#include "webseed_controller.h" + +#include "config.h" + +#include + +#include "utils/log.h" +#include +#include "torrent/utils/log.h" +#include "protocol/peer_chunks.h" +#include "download/download_main.h" +#include "download/chunk_selector.h" +#include "torrent/data/file_list.h" +#include "torrent/data/block.h" +#include "torrent/data/block_list.h" +#include "torrent/throttle.h" +#include "data/chunk_list.h" +#include "data/hash_chunk.h" +#include "torrent/data/file_list.h" +#include "protocol/request_list.h" +#include "manager.h" +#include "torrent/download/download_manager.h" +#include "download/download_wrapper.h" + +namespace torrent { + +WebseedController::WebseedController(DownloadMain* download) : m_download(download) +{ +} + +WebseedController::~WebseedController() { + m_thread_stop = true; + if (m_thread.joinable()) m_thread.join(); +} + +void +WebseedController::add_url(const url_type& url) { + m_url_list.push_back(url); +} + +struct MemoryStruct { + char* memory; + size_t size; +}; + +static size_t +WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + struct MemoryStruct *mem = (struct MemoryStruct *)userp; + + memcpy(&(mem->memory[mem->size]), contents, realsize); + + mem->size += realsize; + + return realsize; +} + +void webseed_dl +( + char* buffer, + const std::string& url, + const int offset, + const int length +) +{ + int end = offset + length - 1; + std::string range_str = std::to_string(offset) + "-" + std::to_string(end); + + lt_log_print(LOG_DEBUG, "webseed: curl %s %s", url.c_str(), range_str.c_str()); + + struct MemoryStruct chunk; + chunk.memory = buffer; + chunk.size = 0; + + CURL* curl = curl_easy_init(); + if (!curl) { + lt_log_print(LOG_ERROR, "webseed: curl init failed"); + } else { + curl_easy_setopt(curl, CURLOPT_RANGE, range_str.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk); + + CURLcode res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + lt_log_print(LOG_ERROR, "webseed: curl failed to retreive %s: %s", url.c_str(), curl_easy_strerror(res)); + } else { + lt_log_print(LOG_DEBUG, "webseed: curl received %u bytes", chunk.size); + } + + /* always cleanup */ + curl_easy_cleanup(curl); + } + +} + +void WebseedController::start() +{ + if(!url_list()->empty()) { + m_thread_stop = false; + m_thread = std::thread(&WebseedController::doWebseed, this); + } +} + +void WebseedController::stop() +{ + m_thread_stop = true; + if (m_thread.joinable()) m_thread.join(); +} + +void WebseedController::doWebseed() +{ + lt_log_print(LOG_INFO, "webseed: thread started for %s", m_download->info()->name().c_str()); + + //webseeds have all chunks + PeerChunks peerChunks; + peerChunks.bitfield()->copy(*m_download->file_list()->bitfield()); + peerChunks.bitfield()->set_all(); + + uint32_t chunk_index; + while (!m_thread_stop && + (chunk_index = m_download->chunk_selector()->find(&peerChunks, false)) != Piece::invalid_index) + { + uint32_t chunk_size = m_download->file_list()->chunk_index_size(chunk_index); + Piece piece(chunk_index, 0, chunk_size); + + TransferList::iterator blockListItr = m_download->delegator()->transfer_list()->insert(piece, Delegator::block_size); + + lt_log_print(LOG_DEBUG, "webseed: getting chunk_index %u size %u", chunk_index, chunk_size); + + static const int chunk_flags = ChunkList::get_writable; + ChunkHandle chunk_handle = m_download->chunk_list()->get(chunk_index, chunk_flags); + + const std::string root_dir = m_download->file_list()->root_dir(); + + for ( + Chunk::const_iterator chunk_part_itr = chunk_handle.chunk()->begin(), + chunk_part_last = chunk_handle.chunk()->end(); + chunk_part_itr != chunk_part_last; + chunk_part_itr++) + { + const ChunkPart& chunk_part = *chunk_part_itr; + + const std::string absolute_file_path = chunk_part.file()->frozen_path(); + int file_offset = chunk_part.file_offset(); + int chunk_part_offset = chunk_part.position(); + int chunk_part_size = chunk_part.size(); + char* chunk_part_buffer = chunk_part.chunk().begin(); + + std::string relative_file_path = absolute_file_path; + relative_file_path.replace(relative_file_path.find(root_dir),root_dir.length(),""); + + lt_log_print(LOG_DEBUG, "webseed: chunk_part mapped to file %s offset %u length %u", + relative_file_path.c_str(), file_offset, chunk_part_size); + + + std::string http_url = url_list()->front() + m_download->info()->name() + relative_file_path; + webseed_dl(chunk_part_buffer, http_url, file_offset, chunk_part_size); + + m_download->info()->mutable_down_rate()->insert(chunk_part_size); + m_download->download_throttle()->node_used(peerChunks.download_throttle(), chunk_part_size); + + } + + m_download->delegator()->transfer_list()->erase(blockListItr); + + + DownloadWrapper* wrapper = *manager->download_manager()->find(m_download->info()); + + static const int SHA1_LENGTH = 20; + + char hash[SHA1_LENGTH]; + HashChunk hash_chunk(chunk_handle); + hash_chunk.perform(hash_chunk.remaining(), false); + hash_chunk.hash_c(&hash[0]); + lt_log_print(LOG_DEBUG, "webseed: chunk hash check. should be %s was %s", + hash_string_to_hex_str(*HashString::cast_from(wrapper->chunk_hash(chunk_handle.index()))).c_str(), + hash_string_to_hex_str(*HashString::cast_from(hash)).c_str()); + + if (std::memcmp(hash, wrapper->chunk_hash(chunk_index), SHA1_LENGTH) == 0) + m_download->file_list()->mark_completed(chunk_index); + + if (m_download->file_list()->is_done()) + if (!m_download->delay_download_done().is_queued()) + priority_queue_insert(&taskScheduler, &m_download->delay_download_done(), cachedTime); + + m_download->chunk_list()->release(&chunk_handle, chunk_flags); + + + } +} + + +} diff --git a/src/torrent/webseed_controller.h b/src/torrent/webseed_controller.h new file mode 100644 index 000000000..eed1da083 --- /dev/null +++ b/src/torrent/webseed_controller.h @@ -0,0 +1,78 @@ +// libTorrent - BitTorrent library +// Copyright (C) 2005-2011, Jari Sundell +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations +// including the two. +// +// You must obey the GNU General Public License in all respects for +// all of the code used other than OpenSSL. If you modify file(s) +// with this exception, you may extend this exception to your version +// of the file(s), but you are not obligated to do so. If you do not +// wish to do so, delete this exception statement from your version. +// If you delete this exception statement from all source files in the +// program, then also delete it here. +// +// Contact: Jari Sundell +// +// Skomakerveien 33 +// 3185 Skoppum, NORWAY + +#ifndef LIBTORRENT_WEBSEED_CONTROLLER_H +#define LIBTORRENT_WEBSEED_CONTROLLER_H + +#include +#include +#include +#include + +#include + +namespace torrent { + +struct tracker_controller_private; + +class LIBTORRENT_EXPORT WebseedController { +public: + typedef std::string url_type; + typedef std::list url_list_type; + + WebseedController(DownloadMain* download); + ~WebseedController(); + + void add_url(const url_type& url); + const url_list_type* url_list() const { return &m_url_list; } + + void start(); + void stop(); + + void doWebseed(); + +private: + DownloadMain* m_download; + + url_list_type m_url_list; + + std::atomic_bool m_thread_stop; + std::thread m_thread; +}; + +} + +#endif diff --git a/src/tracker/Makefile.am b/src/tracker/Makefile.am index 2f1ae5cf5..1847c5cce 100644 --- a/src/tracker/Makefile.am +++ b/src/tracker/Makefile.am @@ -6,6 +6,8 @@ libsub_tracker_la_SOURCES = \ tracker_http.cc \ tracker_http.h \ tracker_udp.cc \ - tracker_udp.h + tracker_udp.h \ + tracker_webseed.cc \ + tracker_webseed.h AM_CPPFLAGS = -I$(srcdir) -I$(srcdir)/.. -I$(top_srcdir) diff --git a/src/tracker/tracker_webseed.cc b/src/tracker/tracker_webseed.cc new file mode 100644 index 000000000..1761a335d --- /dev/null +++ b/src/tracker/tracker_webseed.cc @@ -0,0 +1,109 @@ +// libTorrent - BitTorrent library +// Copyright (C) 2005-2011, Jari Sundell +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations +// including the two. +// +// You must obey the GNU General Public License in all respects for +// all of the code used other than OpenSSL. If you modify file(s) +// with this exception, you may extend this exception to your version +// of the file(s), but you are not obligated to do so. If you do not +// wish to do so, delete this exception statement from your version. +// If you delete this exception statement from all source files in the +// program, then also delete it here. +// +// Contact: Jari Sundell +// +// Skomakerveien 33 +// 3185 Skoppum, NORWAY + +#include "config.h" + +#include +#include + +#include "dht/dht_router.h" +#include "torrent/connection_manager.h" +#include "torrent/download_info.h" +#include "torrent/dht_manager.h" +#include "torrent/exceptions.h" +#include "torrent/tracker_list.h" +#include "torrent/utils/log.h" + +#include "tracker_webseed.h" + +#include "globals.h" +#include "manager.h" + +namespace torrent { + +const char* TrackerWebseed::states[] = { "Idle", "Downloading" }; + +TrackerWebseed::TrackerWebseed(TrackerList* parent, const std::string& url, int flags) : + Tracker(parent, url, flags), + m_state(state_idle) { +} + +TrackerWebseed::~TrackerWebseed() { +} + +bool +TrackerWebseed::is_busy() const { + return m_state != state_idle; +} + +bool +TrackerWebseed::is_usable() const { + return true; +} + +void +TrackerWebseed::send_state(int state) { + if (m_parent == NULL) + throw internal_error("TrackerWebseed::send_state(...) does not have a valid m_parent."); + + m_latest_event = state; + + if (state == DownloadInfo::STOPPED) + return; + +} + +void +TrackerWebseed::close() { +} + +void +TrackerWebseed::disown() { + close(); +} + +TrackerWebseed::Type +TrackerWebseed::type() const { + return TRACKER_WEBSEED; +} + +void +TrackerWebseed::get_status(char* buffer, int length) { + if (!is_busy()) + throw internal_error("TrackerWebseed::get_status called while not busy."); +} + +} diff --git a/src/tracker/tracker_webseed.h b/src/tracker/tracker_webseed.h new file mode 100644 index 000000000..c59f7d64b --- /dev/null +++ b/src/tracker/tracker_webseed.h @@ -0,0 +1,75 @@ +// libTorrent - BitTorrent library +// Copyright (C) 2005-2011, Jari Sundell +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations +// including the two. +// +// You must obey the GNU General Public License in all respects for +// all of the code used other than OpenSSL. If you modify file(s) +// with this exception, you may extend this exception to your version +// of the file(s), but you are not obligated to do so. If you do not +// wish to do so, delete this exception statement from your version. +// If you delete this exception statement from all source files in the +// program, then also delete it here. +// +// Contact: Jari Sundell +// +// Skomakerveien 33 +// 3185 Skoppum, NORWAY + +#ifndef LIBTORRENT_TRACKER_TRACKER_WEBSEED_H +#define LIBTORRENT_TRACKER_TRACKER_WEBSEED_H + +#include "net/address_list.h" +#include "torrent/object.h" +#include "torrent/tracker.h" + +namespace torrent { + +class TrackerWebseed : public Tracker { +public: + TrackerWebseed(TrackerList* parent, const std::string& url, int flags); + ~TrackerWebseed(); + + typedef enum { + state_idle, + state_downloading, + } state_type; + + static const char* states[]; + + virtual bool is_busy() const; + virtual bool is_usable() const; + + virtual void send_state(int state); + virtual void close(); + virtual void disown(); + + virtual Type type() const; + + virtual void get_status(char* buffer, int length); + +private: + state_type m_state; +}; + +} + +#endif From c13ae05bc9ab2dbaac1d5e7ea9c2d464533c8427 Mon Sep 17 00:00:00 2001 From: naf Date: Mon, 5 Feb 2018 13:07:18 -0600 Subject: [PATCH 2/7] add curl dependency for tests --- configure.ac | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 65e348721..0471a5f2a 100644 --- a/configure.ac +++ b/configure.ac @@ -60,10 +60,11 @@ AX_PTHREAD AX_CHECK_ZLIB PKG_CHECK_MODULES([CPPUNIT], [cppunit],, [no_cppunit="yes"]) +PKG_CHECK_MODULES([LIBCURL], [libcurl >= 7.15.4], , [no_libcurl="yes"]) -CFLAGS="$PTHREAD_CFLAGS $CPPUNIT_CFLAGS $CFLAGS" -CXXFLAGS="$PTHREAD_CFLAGS $CPPUNIT_CFLAGS $CXXFLAGS" -LIBS="$PTHREAD_LIBS $CPPUNIT_LIBS $LIBS" +CFLAGS="$PTHREAD_CFLAGS $CPPUNIT_CFLAGS $LIBCURL_CFLAGS $CFLAGS" +CXXFLAGS="$PTHREAD_CFLAGS $CPPUNIT_CFLAGS $LIBCURL_CXXFLAGS $CXXFLAGS" +LIBS="$PTHREAD_LIBS $CPPUNIT_LIBS $LIBCURL_LIBS $LIBS" AC_ARG_ENABLE(openssl, [ --disable-openssl Don't use OpenSSL's SHA1 implementation.], From 051f3679ab7d2dc280d2bb571cc3bd416b73b060 Mon Sep 17 00:00:00 2001 From: naf Date: Mon, 5 Feb 2018 20:29:10 -0600 Subject: [PATCH 3/7] make code readable --- src/torrent/webseed_controller.cc | 117 +++++++++++++----------------- src/torrent/webseed_controller.h | 18 ++++- 2 files changed, 67 insertions(+), 68 deletions(-) diff --git a/src/torrent/webseed_controller.cc b/src/torrent/webseed_controller.cc index 73411c21a..ff76ee522 100644 --- a/src/torrent/webseed_controller.cc +++ b/src/torrent/webseed_controller.cc @@ -60,7 +60,7 @@ namespace torrent { -WebseedController::WebseedController(DownloadMain* download) : m_download(download) +WebseedController::WebseedController(DownloadMain* download) : m_download(download), m_allChunks(new PeerChunks) { } @@ -69,8 +69,7 @@ WebseedController::~WebseedController() { if (m_thread.joinable()) m_thread.join(); } -void -WebseedController::add_url(const url_type& url) { +void WebseedController::add_url(const url_type& url) { m_url_list.push_back(url); } @@ -78,10 +77,9 @@ struct MemoryStruct { char* memory; size_t size; }; - + static size_t -WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) -{ +WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) { size_t realsize = size * nmemb; struct MemoryStruct *mem = (struct MemoryStruct *)userp; @@ -92,7 +90,7 @@ WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) return realsize; } -void webseed_dl +void WebseedController::download_to_buffer ( char* buffer, const std::string& url, @@ -106,7 +104,7 @@ void webseed_dl lt_log_print(LOG_DEBUG, "webseed: curl %s %s", url.c_str(), range_str.c_str()); struct MemoryStruct chunk; - chunk.memory = buffer; + chunk.memory = buffer; chunk.size = 0; CURL* curl = curl_easy_init(); @@ -153,82 +151,69 @@ void WebseedController::doWebseed() lt_log_print(LOG_INFO, "webseed: thread started for %s", m_download->info()->name().c_str()); //webseeds have all chunks - PeerChunks peerChunks; - peerChunks.bitfield()->copy(*m_download->file_list()->bitfield()); - peerChunks.bitfield()->set_all(); + m_allChunks->bitfield()->copy(*m_download->file_list()->bitfield()); + m_allChunks->bitfield()->set_all(); uint32_t chunk_index; while (!m_thread_stop && - (chunk_index = m_download->chunk_selector()->find(&peerChunks, false)) != Piece::invalid_index) + (chunk_index = m_download->chunk_selector()->find(&(*m_allChunks), false)) != Piece::invalid_index) { - uint32_t chunk_size = m_download->file_list()->chunk_index_size(chunk_index); - Piece piece(chunk_index, 0, chunk_size); - - TransferList::iterator blockListItr = m_download->delegator()->transfer_list()->insert(piece, Delegator::block_size); - - lt_log_print(LOG_DEBUG, "webseed: getting chunk_index %u size %u", chunk_index, chunk_size); - - static const int chunk_flags = ChunkList::get_writable; - ChunkHandle chunk_handle = m_download->chunk_list()->get(chunk_index, chunk_flags); - - const std::string root_dir = m_download->file_list()->root_dir(); - - for ( - Chunk::const_iterator chunk_part_itr = chunk_handle.chunk()->begin(), - chunk_part_last = chunk_handle.chunk()->end(); - chunk_part_itr != chunk_part_last; - chunk_part_itr++) - { - const ChunkPart& chunk_part = *chunk_part_itr; - - const std::string absolute_file_path = chunk_part.file()->frozen_path(); - int file_offset = chunk_part.file_offset(); - int chunk_part_offset = chunk_part.position(); - int chunk_part_size = chunk_part.size(); - char* chunk_part_buffer = chunk_part.chunk().begin(); - - std::string relative_file_path = absolute_file_path; - relative_file_path.replace(relative_file_path.find(root_dir),root_dir.length(),""); + download_chunk(chunk_index); + } +} - lt_log_print(LOG_DEBUG, "webseed: chunk_part mapped to file %s offset %u length %u", - relative_file_path.c_str(), file_offset, chunk_part_size); +void WebseedController::download_chunk(const uint32_t chunk_index) +{ + uint32_t chunk_size = m_download->file_list()->chunk_index_size(chunk_index); + Piece piece(chunk_index, 0, chunk_size); + TransferList::iterator blockListItr = m_download->delegator()->transfer_list()->insert(piece, Delegator::block_size); - std::string http_url = url_list()->front() + m_download->info()->name() + relative_file_path; - webseed_dl(chunk_part_buffer, http_url, file_offset, chunk_part_size); + lt_log_print(LOG_DEBUG, "webseed: getting chunk_index %u size %u", chunk_index, chunk_size); - m_download->info()->mutable_down_rate()->insert(chunk_part_size); - m_download->download_throttle()->node_used(peerChunks.download_throttle(), chunk_part_size); + static const int chunk_flags = ChunkList::get_writable; + ChunkHandle chunk_handle = m_download->chunk_list()->get(chunk_index, chunk_flags); - } - - m_download->delegator()->transfer_list()->erase(blockListItr); + std::for_each(chunk_handle.chunk()->begin(), chunk_handle.chunk()->end(), + std::bind(&WebseedController::download_chunk_part, this, std::placeholders::_1)); + m_download->delegator()->transfer_list()->erase(blockListItr); - DownloadWrapper* wrapper = *manager->download_manager()->find(m_download->info()); + DownloadWrapper* wrapper = *manager->download_manager()->find(m_download->info()); - static const int SHA1_LENGTH = 20; + char hash[HashString::size_data]; + HashChunk hash_chunk(chunk_handle); + hash_chunk.perform(hash_chunk.remaining(), false); + hash_chunk.hash_c(&hash[0]); + lt_log_print(LOG_DEBUG, "webseed: chunk hash check. should be %s was %s", + hash_string_to_hex_str(*HashString::cast_from(wrapper->chunk_hash(chunk_handle.index()))).c_str(), + hash_string_to_hex_str(*HashString::cast_from(hash)).c_str()); - char hash[SHA1_LENGTH]; - HashChunk hash_chunk(chunk_handle); - hash_chunk.perform(hash_chunk.remaining(), false); - hash_chunk.hash_c(&hash[0]); - lt_log_print(LOG_DEBUG, "webseed: chunk hash check. should be %s was %s", - hash_string_to_hex_str(*HashString::cast_from(wrapper->chunk_hash(chunk_handle.index()))).c_str(), - hash_string_to_hex_str(*HashString::cast_from(hash)).c_str()); + if (std::memcmp(hash, wrapper->chunk_hash(chunk_index), HashString::size_data) == 0) + m_download->file_list()->mark_completed(chunk_index); - if (std::memcmp(hash, wrapper->chunk_hash(chunk_index), SHA1_LENGTH) == 0) - m_download->file_list()->mark_completed(chunk_index); + if (m_download->file_list()->is_done()) + if (!m_download->delay_download_done().is_queued()) + priority_queue_insert(&taskScheduler, &m_download->delay_download_done(), cachedTime); - if (m_download->file_list()->is_done()) - if (!m_download->delay_download_done().is_queued()) - priority_queue_insert(&taskScheduler, &m_download->delay_download_done(), cachedTime); + m_download->chunk_list()->release(&chunk_handle, chunk_flags); +} - m_download->chunk_list()->release(&chunk_handle, chunk_flags); +void WebseedController::download_chunk_part(const ChunkPart& chunk_part) +{ + const std::string& absolute_file_path = chunk_part.file()->frozen_path(); + const std::string& root_dir = m_download->file_list()->root_dir(); + std::string relative_file_path = absolute_file_path; + relative_file_path.replace(relative_file_path.find(root_dir),root_dir.length(),""); + lt_log_print(LOG_DEBUG, "webseed: chunk_part mapped to file %s offset %u length %u", + relative_file_path.c_str(), chunk_part.file_offset(), chunk_part.size()); - } -} + std::string http_url = url_list()->front() + m_download->info()->name() + relative_file_path; + download_to_buffer(chunk_part.chunk().begin(), http_url, chunk_part.file_offset(), chunk_part.size()); + m_download->info()->mutable_down_rate()->insert(chunk_part.size()); + m_download->download_throttle()->node_used(m_allChunks->download_throttle(), chunk_part.size()); +} } diff --git a/src/torrent/webseed_controller.h b/src/torrent/webseed_controller.h index eed1da083..df2c430aa 100644 --- a/src/torrent/webseed_controller.h +++ b/src/torrent/webseed_controller.h @@ -41,12 +41,14 @@ #include #include #include +#include #include namespace torrent { -struct tracker_controller_private; +class ChunkPart; +class PeerChunks; class LIBTORRENT_EXPORT WebseedController { public: @@ -65,12 +67,24 @@ class LIBTORRENT_EXPORT WebseedController { void doWebseed(); private: - DownloadMain* m_download; + void download_to_buffer + ( + char* buffer, + const std::string& url, + const int offset, + const int length + ); + + void download_chunk_part(const ChunkPart& chunk_part); + void download_chunk(const uint32_t chunk_index); + DownloadMain* m_download; url_list_type m_url_list; std::atomic_bool m_thread_stop; std::thread m_thread; + + std::unique_ptr m_allChunks; }; } From 1b972f2c0921f57336eb42effff8891fe46ee1c1 Mon Sep 17 00:00:00 2001 From: naf Date: Tue, 6 Feb 2018 08:53:10 -0600 Subject: [PATCH 4/7] add trivial tracker test --- test/Makefile.am | 2 ++ test/tracker/tracker_webseed_test.cc | 35 ++++++++++++++++++++++++++++ test/tracker/tracker_webseed_test.h | 15 ++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 test/tracker/tracker_webseed_test.cc create mode 100644 test/tracker/tracker_webseed_test.h diff --git a/test/Makefile.am b/test/Makefile.am index d7a9d5b3b..a0e6a328d 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -80,6 +80,8 @@ LibTorrentTest_SOURCES = \ torrent/tracker_timeout_test.h \ tracker/tracker_http_test.cc \ tracker/tracker_http_test.h \ + tracker/tracker_webseed_test.cc \ + tracker/tracker_webseed_test.h \ main.cc LibTorrentTest_CXXFLAGS = $(CPPUNIT_CFLAGS) diff --git a/test/tracker/tracker_webseed_test.cc b/test/tracker/tracker_webseed_test.cc new file mode 100644 index 000000000..1785df1ba --- /dev/null +++ b/test/tracker/tracker_webseed_test.cc @@ -0,0 +1,35 @@ +#include "config.h" + +#include "tracker_webseed_test.h" + +#include "tracker/tracker_webseed.h" +#include "torrent/tracker_list.h" + +CPPUNIT_TEST_SUITE_REGISTRATION(TrackerWebseedTest); + +void +TrackerWebseedTest::setUp() { +} + +void +TrackerWebseedTest::tearDown() { +} + +void +TrackerWebseedTest::test_basic() { + + torrent::TrackerList parent; + std::string url; + int flags; + + torrent::TrackerWebseed tracker(&parent, url, flags); + + CPPUNIT_ASSERT(tracker.is_busy() == false); + CPPUNIT_ASSERT(tracker.is_usable() == true); + + tracker.close(); + tracker.disown(); + + CPPUNIT_ASSERT(tracker.type() == torrent::Tracker::TRACKER_WEBSEED); + +} diff --git a/test/tracker/tracker_webseed_test.h b/test/tracker/tracker_webseed_test.h new file mode 100644 index 000000000..d5e59aa81 --- /dev/null +++ b/test/tracker/tracker_webseed_test.h @@ -0,0 +1,15 @@ +#include + +#include "tracker/tracker_webseed.h" + +class TrackerWebseedTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TrackerWebseedTest); + CPPUNIT_TEST(test_basic); + CPPUNIT_TEST_SUITE_END(); + +public: + void setUp(); + void tearDown(); + + void test_basic(); +}; From ccffe17fcffd3b5d51a172bce0ad12c4be17d566 Mon Sep 17 00:00:00 2001 From: naf Date: Tue, 6 Feb 2018 10:58:58 -0600 Subject: [PATCH 5/7] add trivial test for webseed_controller --- src/torrent/webseed_controller.h | 4 +++ test/Makefile.am | 2 ++ test/torrent/webseed_controller_test.cc | 35 +++++++++++++++++++++++++ test/torrent/webseed_controller_test.h | 13 +++++++++ 4 files changed, 54 insertions(+) create mode 100644 test/torrent/webseed_controller_test.cc create mode 100644 test/torrent/webseed_controller_test.h diff --git a/src/torrent/webseed_controller.h b/src/torrent/webseed_controller.h index df2c430aa..8da420433 100644 --- a/src/torrent/webseed_controller.h +++ b/src/torrent/webseed_controller.h @@ -45,12 +45,16 @@ #include +class WebseedControllerTest; + namespace torrent { class ChunkPart; class PeerChunks; class LIBTORRENT_EXPORT WebseedController { + friend WebseedControllerTest; + public: typedef std::string url_type; typedef std::list url_list_type; diff --git a/test/Makefile.am b/test/Makefile.am index a0e6a328d..bcadd8f9d 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -66,6 +66,8 @@ LibTorrentTest_SOURCES = \ torrent/object_static_map_test.h \ torrent/object_stream_test.cc \ torrent/object_stream_test.h \ + torrent/webseed_controller_test.cc \ + torrent/webseed_controller_test.h \ torrent/tracker_controller_test.cc \ torrent/tracker_controller_test.h \ torrent/tracker_controller_features.cc \ diff --git a/test/torrent/webseed_controller_test.cc b/test/torrent/webseed_controller_test.cc new file mode 100644 index 000000000..9b31eddd5 --- /dev/null +++ b/test/torrent/webseed_controller_test.cc @@ -0,0 +1,35 @@ +#include "config.h" + +#include "webseed_controller_test.h" + +#include "torrent/webseed_controller.h" + +CPPUNIT_TEST_SUITE_REGISTRATION(WebseedControllerTest); + +void +WebseedControllerTest::setUp() { +} + +void +WebseedControllerTest::tearDown() { +} + + +namespace torrent{ + class Manager {}; + + Manager manager; +} + +void +WebseedControllerTest::test_basic() { + torrent::WebseedController controller(NULL); + + CPPUNIT_ASSERT(controller.url_list()->size() == 0); + controller.add_url("http://test/test"); + CPPUNIT_ASSERT(controller.url_list()->size() == 1); + + char* buffer = (char*)malloc(1000); + controller.download_to_buffer(buffer, "file://blah", 0, 1000); + free(buffer); +} diff --git a/test/torrent/webseed_controller_test.h b/test/torrent/webseed_controller_test.h new file mode 100644 index 000000000..fd87ffbc4 --- /dev/null +++ b/test/torrent/webseed_controller_test.h @@ -0,0 +1,13 @@ +#include + +class WebseedControllerTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(WebseedControllerTest); + CPPUNIT_TEST(test_basic); + CPPUNIT_TEST_SUITE_END(); + +public: + void setUp(); + void tearDown(); + + void test_basic(); +}; From e3ab174f27f2e75f77d020603d10ce976257ca65 Mon Sep 17 00:00:00 2001 From: naf Date: Wed, 7 Feb 2018 11:28:02 -0600 Subject: [PATCH 6/7] add error handling --- src/torrent/webseed_controller.cc | 203 +++++++++++++++++------------- src/torrent/webseed_controller.h | 25 +++- 2 files changed, 134 insertions(+), 94 deletions(-) diff --git a/src/torrent/webseed_controller.cc b/src/torrent/webseed_controller.cc index ff76ee522..3031a7a4d 100644 --- a/src/torrent/webseed_controller.cc +++ b/src/torrent/webseed_controller.cc @@ -39,9 +39,11 @@ #include "config.h" #include +#include -#include "utils/log.h" #include + +#include "utils/log.h" #include "torrent/utils/log.h" #include "protocol/peer_chunks.h" #include "download/download_main.h" @@ -52,7 +54,6 @@ #include "torrent/throttle.h" #include "data/chunk_list.h" #include "data/hash_chunk.h" -#include "torrent/data/file_list.h" #include "protocol/request_list.h" #include "manager.h" #include "torrent/download/download_manager.h" @@ -73,65 +74,6 @@ void WebseedController::add_url(const url_type& url) { m_url_list.push_back(url); } -struct MemoryStruct { - char* memory; - size_t size; -}; - -static size_t -WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) { - size_t realsize = size * nmemb; - struct MemoryStruct *mem = (struct MemoryStruct *)userp; - - memcpy(&(mem->memory[mem->size]), contents, realsize); - - mem->size += realsize; - - return realsize; -} - -void WebseedController::download_to_buffer -( - char* buffer, - const std::string& url, - const int offset, - const int length -) -{ - int end = offset + length - 1; - std::string range_str = std::to_string(offset) + "-" + std::to_string(end); - - lt_log_print(LOG_DEBUG, "webseed: curl %s %s", url.c_str(), range_str.c_str()); - - struct MemoryStruct chunk; - chunk.memory = buffer; - chunk.size = 0; - - CURL* curl = curl_easy_init(); - if (!curl) { - lt_log_print(LOG_ERROR, "webseed: curl init failed"); - } else { - curl_easy_setopt(curl, CURLOPT_RANGE, range_str.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk); - - CURLcode res = curl_easy_perform(curl); - /* Check for errors */ - if(res != CURLE_OK) { - lt_log_print(LOG_ERROR, "webseed: curl failed to retreive %s: %s", url.c_str(), curl_easy_strerror(res)); - } else { - lt_log_print(LOG_DEBUG, "webseed: curl received %u bytes", chunk.size); - } - - /* always cleanup */ - curl_easy_cleanup(curl); - } - -} - void WebseedController::start() { if(!url_list()->empty()) { @@ -154,52 +96,71 @@ void WebseedController::doWebseed() m_allChunks->bitfield()->copy(*m_download->file_list()->bitfield()); m_allChunks->bitfield()->set_all(); + m_working_url_list = m_url_list; + std::random_shuffle(m_working_url_list.begin(), m_working_url_list.end()); + uint32_t chunk_index; - while (!m_thread_stop && + while (!m_thread_stop && + !m_working_url_list.empty() && (chunk_index = m_download->chunk_selector()->find(&(*m_allChunks), false)) != Piece::invalid_index) { - download_chunk(chunk_index); + std::string url = m_working_url_list.front(); + + bool success = download_chunk(chunk_index, url); + + if (!success) + { + m_working_url_list.erase(m_working_url_list.begin()); + } } -} +} -void WebseedController::download_chunk(const uint32_t chunk_index) +bool WebseedController::download_chunk(const uint32_t chunk_index, const std::string& url_base) { uint32_t chunk_size = m_download->file_list()->chunk_index_size(chunk_index); Piece piece(chunk_index, 0, chunk_size); - TransferList::iterator blockListItr = m_download->delegator()->transfer_list()->insert(piece, Delegator::block_size); - - lt_log_print(LOG_DEBUG, "webseed: getting chunk_index %u size %u", chunk_index, chunk_size); - static const int chunk_flags = ChunkList::get_writable; ChunkHandle chunk_handle = m_download->chunk_list()->get(chunk_index, chunk_flags); + TransferList::iterator blockListItr = m_download->delegator()->transfer_list()->insert(piece, Delegator::block_size); + + lt_log_print(LOG_DEBUG, "webseed: getting chunk_index %u size %u", chunk_index, chunk_size); + + bool success = true; std::for_each(chunk_handle.chunk()->begin(), chunk_handle.chunk()->end(), - std::bind(&WebseedController::download_chunk_part, this, std::placeholders::_1)); + [&](const ChunkPart& part){ success &= download_chunk_part(part, url_base); }); m_download->delegator()->transfer_list()->erase(blockListItr); - DownloadWrapper* wrapper = *manager->download_manager()->find(m_download->info()); - - char hash[HashString::size_data]; - HashChunk hash_chunk(chunk_handle); - hash_chunk.perform(hash_chunk.remaining(), false); - hash_chunk.hash_c(&hash[0]); - lt_log_print(LOG_DEBUG, "webseed: chunk hash check. should be %s was %s", - hash_string_to_hex_str(*HashString::cast_from(wrapper->chunk_hash(chunk_handle.index()))).c_str(), - hash_string_to_hex_str(*HashString::cast_from(hash)).c_str()); - - if (std::memcmp(hash, wrapper->chunk_hash(chunk_index), HashString::size_data) == 0) - m_download->file_list()->mark_completed(chunk_index); - - if (m_download->file_list()->is_done()) - if (!m_download->delay_download_done().is_queued()) - priority_queue_insert(&taskScheduler, &m_download->delay_download_done(), cachedTime); + if (success) + { + DownloadWrapper* wrapper = *manager->download_manager()->find(m_download->info()); + + char hash[HashString::size_data]; + HashChunk hash_chunk(chunk_handle); + hash_chunk.perform(hash_chunk.remaining(), false); + hash_chunk.hash_c(&hash[0]); + lt_log_print(LOG_DEBUG, "webseed: chunk hash check. should be %s was %s", + hash_string_to_hex_str(*HashString::cast_from(wrapper->chunk_hash(chunk_handle.index()))).c_str(), + hash_string_to_hex_str(*HashString::cast_from(hash)).c_str()); + + if (std::memcmp(hash, wrapper->chunk_hash(chunk_index), HashString::size_data) == 0) + m_download->file_list()->mark_completed(chunk_index); + else + success = false; + + if (m_download->file_list()->is_done()) + if (!m_download->delay_download_done().is_queued()) + priority_queue_insert(&taskScheduler, &m_download->delay_download_done(), cachedTime); + } m_download->chunk_list()->release(&chunk_handle, chunk_flags); + + return success; } -void WebseedController::download_chunk_part(const ChunkPart& chunk_part) +bool WebseedController::download_chunk_part(const ChunkPart& chunk_part, const std::string& url_base) { const std::string& absolute_file_path = chunk_part.file()->frozen_path(); const std::string& root_dir = m_download->file_list()->root_dir(); @@ -209,11 +170,75 @@ void WebseedController::download_chunk_part(const ChunkPart& chunk_part) lt_log_print(LOG_DEBUG, "webseed: chunk_part mapped to file %s offset %u length %u", relative_file_path.c_str(), chunk_part.file_offset(), chunk_part.size()); - std::string http_url = url_list()->front() + m_download->info()->name() + relative_file_path; - download_to_buffer(chunk_part.chunk().begin(), http_url, chunk_part.file_offset(), chunk_part.size()); + std::string http_url = url_base + m_download->info()->name() + relative_file_path; + int success = download_to_buffer(chunk_part.chunk().begin(), http_url, chunk_part.file_offset(), chunk_part.size()); m_download->info()->mutable_down_rate()->insert(chunk_part.size()); m_download->download_throttle()->node_used(m_allChunks->download_throttle(), chunk_part.size()); + + return success; } +bool WebseedController::download_to_buffer +( + char* buffer, + const std::string& url, + const int offset, + const int length +) +{ + bool success = false; + + int end = offset + length - 1; + std::string range_str = std::to_string(offset) + "-" + std::to_string(end); + + lt_log_print(LOG_DEBUG, "webseed: curl %s %s", url.c_str(), range_str.c_str()); + + WebseedDownloadBuffer downloadBuffer(buffer, length); + + CURL* curl = curl_easy_init(); + if (!curl) { + lt_log_print(LOG_ERROR, "webseed: curl init failed"); + } else { + curl_easy_setopt(curl, CURLOPT_RANGE, range_str.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WebseedDownloadBuffer::receiveBytesCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&downloadBuffer); + + CURLcode res = curl_easy_perform(curl); + + if(res != CURLE_OK) { + lt_log_print(LOG_ERROR, "webseed: curl failed to retreive %s: %s", url.c_str(), curl_easy_strerror(res)); + } else { + lt_log_print(LOG_DEBUG, "webseed: curl received %u bytes", downloadBuffer.getBytesWritten()); + success = true; + } + + curl_easy_cleanup(curl); + } + + return success; +} + + +WebseedDownloadBuffer::WebseedDownloadBuffer(char* buffer, size_t size) : + m_buffer(buffer), m_buffer_size(size), m_position(0) +{ } + +size_t WebseedDownloadBuffer::receiveBytesCallback(void *contents, size_t size, size_t nmemb, void *userp) { + WebseedDownloadBuffer* downloadBuffer = (WebseedDownloadBuffer*)userp; + return downloadBuffer->receiveBytes(contents, size * nmemb); +} + +size_t WebseedDownloadBuffer::receiveBytes(void* contents, size_t realsize) +{ + if (m_position + realsize > m_buffer_size) return 0; + + memcpy(&(m_buffer[m_position]), contents, realsize); + m_position += realsize; + return realsize; +} + } diff --git a/src/torrent/webseed_controller.h b/src/torrent/webseed_controller.h index 8da420433..ba77bd48f 100644 --- a/src/torrent/webseed_controller.h +++ b/src/torrent/webseed_controller.h @@ -39,7 +39,7 @@ #include #include -#include +#include #include #include @@ -52,12 +52,26 @@ namespace torrent { class ChunkPart; class PeerChunks; +class WebseedDownloadBuffer { +public: + WebseedDownloadBuffer(char* buffer, size_t size); + static size_t receiveBytesCallback(void *contents, size_t size, size_t nmemb, void *userp); + size_t getBytesWritten() { return m_position; } + +private: + size_t receiveBytes(void* contents, size_t realsize); + + char* m_buffer; + size_t m_buffer_size; + size_t m_position; +}; + class LIBTORRENT_EXPORT WebseedController { friend WebseedControllerTest; public: typedef std::string url_type; - typedef std::list url_list_type; + typedef std::vector url_list_type; WebseedController(DownloadMain* download); ~WebseedController(); @@ -71,7 +85,7 @@ class LIBTORRENT_EXPORT WebseedController { void doWebseed(); private: - void download_to_buffer + bool download_to_buffer ( char* buffer, const std::string& url, @@ -79,11 +93,12 @@ class LIBTORRENT_EXPORT WebseedController { const int length ); - void download_chunk_part(const ChunkPart& chunk_part); - void download_chunk(const uint32_t chunk_index); + bool download_chunk_part(const ChunkPart& chunk_part, const std::string& url_base); + bool download_chunk(const uint32_t chunk_index, const std::string& url_base); DownloadMain* m_download; url_list_type m_url_list; + url_list_type m_working_url_list; std::atomic_bool m_thread_stop; std::thread m_thread; From b27e5e69c701e227122372458c499ef6a67b0ab9 Mon Sep 17 00:00:00 2001 From: naf Date: Sun, 18 Feb 2018 17:47:23 -0600 Subject: [PATCH 7/7] fix single-entry url-list and urls ending with / --- src/download/download_constructor.cc | 9 +++---- src/torrent/webseed_controller.cc | 36 ++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/download/download_constructor.cc b/src/download/download_constructor.cc index 61d377b33..8c1c10417 100644 --- a/src/download/download_constructor.cc +++ b/src/download/download_constructor.cc @@ -210,10 +210,8 @@ DownloadConstructor::parse_tracker(const Object& b) { std::for_each(b.get_key_list("nodes").begin(), b.get_key_list("nodes").end(), rak::make_mem_fun(this, &DownloadConstructor::add_dht_node)); - if (b.has_key_list("url-list")) - { - std::for_each(b.get_key_list("url-list").begin(), b.get_key_list("url-list").end(), rak::make_mem_fun(this, &DownloadConstructor::add_webseed_url)); - } + if (b.has_key("url-list")) + add_webseed_url(b.get_key("url-list")); m_download->main()->tracker_list()->randomize_group_entries(); } @@ -256,6 +254,9 @@ DownloadConstructor::add_dht_node(const Object& b) { void DownloadConstructor::add_webseed_url(const Object& b) { + if (b.is_list()) + std::for_each(b.as_list().begin(), b.as_list().end(), rak::make_mem_fun(this, &DownloadConstructor::add_webseed_url)); + if (!b.is_string()) return; diff --git a/src/torrent/webseed_controller.cc b/src/torrent/webseed_controller.cc index 3031a7a4d..3bd515f44 100644 --- a/src/torrent/webseed_controller.cc +++ b/src/torrent/webseed_controller.cc @@ -59,6 +59,9 @@ #include "torrent/download/download_manager.h" #include "download/download_wrapper.h" +#define LT_LOG_WEBSEED(log_level, log_fmt, ...) \ + lt_log_print_info(LOG_TRACKER_##log_level, m_download->info(), "webseed_controller", log_fmt, __VA_ARGS__); + namespace torrent { WebseedController::WebseedController(DownloadMain* download) : m_download(download), m_allChunks(new PeerChunks) @@ -71,7 +74,8 @@ WebseedController::~WebseedController() { } void WebseedController::add_url(const url_type& url) { - m_url_list.push_back(url); + if (!url.empty()) + m_url_list.push_back(url); } void WebseedController::start() @@ -90,7 +94,7 @@ void WebseedController::stop() void WebseedController::doWebseed() { - lt_log_print(LOG_INFO, "webseed: thread started for %s", m_download->info()->name().c_str()); + LT_LOG_WEBSEED(INFO, "thread started for %s", m_download->info()->name().c_str()); //webseeds have all chunks m_allChunks->bitfield()->copy(*m_download->file_list()->bitfield()); @@ -110,9 +114,15 @@ void WebseedController::doWebseed() if (!success) { + LT_LOG_WEBSEED(ERROR, "download_chunk failed. marking url as non-working: %s", url.c_str()); m_working_url_list.erase(m_working_url_list.begin()); } } + + if (m_working_url_list.empty()) + { + LT_LOG_WEBSEED(ERROR, "%s", "no working webseed urls. stopping webseed dl"); + } } bool WebseedController::download_chunk(const uint32_t chunk_index, const std::string& url_base) @@ -125,7 +135,7 @@ bool WebseedController::download_chunk(const uint32_t chunk_index, const std::st TransferList::iterator blockListItr = m_download->delegator()->transfer_list()->insert(piece, Delegator::block_size); - lt_log_print(LOG_DEBUG, "webseed: getting chunk_index %u size %u", chunk_index, chunk_size); + LT_LOG_WEBSEED(DEBUG, "getting chunk_index %u size %u", chunk_index, chunk_size); bool success = true; std::for_each(chunk_handle.chunk()->begin(), chunk_handle.chunk()->end(), @@ -141,7 +151,7 @@ bool WebseedController::download_chunk(const uint32_t chunk_index, const std::st HashChunk hash_chunk(chunk_handle); hash_chunk.perform(hash_chunk.remaining(), false); hash_chunk.hash_c(&hash[0]); - lt_log_print(LOG_DEBUG, "webseed: chunk hash check. should be %s was %s", + LT_LOG_WEBSEED(DEBUG, "chunk hash check. should be %s was %s", hash_string_to_hex_str(*HashString::cast_from(wrapper->chunk_hash(chunk_handle.index()))).c_str(), hash_string_to_hex_str(*HashString::cast_from(hash)).c_str()); @@ -167,10 +177,16 @@ bool WebseedController::download_chunk_part(const ChunkPart& chunk_part, const s std::string relative_file_path = absolute_file_path; relative_file_path.replace(relative_file_path.find(root_dir),root_dir.length(),""); - lt_log_print(LOG_DEBUG, "webseed: chunk_part mapped to file %s offset %u length %u", + LT_LOG_WEBSEED(DEBUG, "chunk_part mapped to file %s offset %u length %u", relative_file_path.c_str(), chunk_part.file_offset(), chunk_part.size()); - std::string http_url = url_base + m_download->info()->name() + relative_file_path; + + std::string http_url = url_base; + if (url_base[url_base.length() - 1] == '/') + http_url += m_download->info()->name(); + if (m_download->file_list()->is_multi_file()) + http_url += relative_file_path; + int success = download_to_buffer(chunk_part.chunk().begin(), http_url, chunk_part.file_offset(), chunk_part.size()); m_download->info()->mutable_down_rate()->insert(chunk_part.size()); @@ -192,13 +208,13 @@ bool WebseedController::download_to_buffer int end = offset + length - 1; std::string range_str = std::to_string(offset) + "-" + std::to_string(end); - lt_log_print(LOG_DEBUG, "webseed: curl %s %s", url.c_str(), range_str.c_str()); + LT_LOG_WEBSEED(DEBUG, "curl %s %s", url.c_str(), range_str.c_str()); WebseedDownloadBuffer downloadBuffer(buffer, length); CURL* curl = curl_easy_init(); if (!curl) { - lt_log_print(LOG_ERROR, "webseed: curl init failed"); + LT_LOG_WEBSEED(ERROR, "%s", "curl init failed"); } else { curl_easy_setopt(curl, CURLOPT_RANGE, range_str.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); @@ -210,9 +226,9 @@ bool WebseedController::download_to_buffer CURLcode res = curl_easy_perform(curl); if(res != CURLE_OK) { - lt_log_print(LOG_ERROR, "webseed: curl failed to retreive %s: %s", url.c_str(), curl_easy_strerror(res)); + LT_LOG_WEBSEED(ERROR, "curl failed to retreive %s: %s", url.c_str(), curl_easy_strerror(res)); } else { - lt_log_print(LOG_DEBUG, "webseed: curl received %u bytes", downloadBuffer.getBytesWritten()); + LT_LOG_WEBSEED(DEBUG, "curl received %u bytes", downloadBuffer.getBytesWritten()); success = true; }