diff --git a/.github/workflows/build-arm64.yml b/.github/workflows/build-arm64.yml index 317e5593..3a797390 100644 --- a/.github/workflows/build-arm64.yml +++ b/.github/workflows/build-arm64.yml @@ -28,7 +28,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: Debug build - path: ${{github.workspace}}/build/chia_plot + path: | + ${{github.workspace}}/build/chia_plot + ${{github.workspace}}/build/chia_plot_k34 build-release: runs-on: [ARM64] @@ -46,7 +48,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: Release build - path: ${{github.workspace}}/build/chia_plot + path: | + ${{github.workspace}}/build/chia_plot + ${{github.workspace}}/build/chia_plot_k34 - name: Get tag name if: startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 6e101466..65be9a22 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -28,7 +28,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: Debug build - path: ${{github.workspace}}/build/chia_plot + path: | + ${{github.workspace}}/build/chia_plot + ${{github.workspace}}/build/chia_plot_k34 build-release: runs-on: ubuntu-latest @@ -46,7 +48,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: Release build - path: ${{github.workspace}}/build/chia_plot + path: | + ${{github.workspace}}/build/chia_plot + ${{github.workspace}}/build/chia_plot_k34 - name: Get tag name if: startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/build-m1.yml b/.github/workflows/build-m1.yml index e2826e78..efdc0bfe 100644 --- a/.github/workflows/build-m1.yml +++ b/.github/workflows/build-m1.yml @@ -31,7 +31,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: Debug build - path: ${{github.workspace}}/build/chia_plot + path: | + ${{github.workspace}}/build/chia_plot + ${{github.workspace}}/build/chia_plot_k34 build-release: runs-on: [m1] @@ -52,7 +54,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: Release build - path: ${{github.workspace}}/build/chia_plot + path: | + ${{github.workspace}}/build/chia_plot + ${{github.workspace}}/build/chia_plot_k34 - name: Get tag name if: startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/build-osx.yml b/.github/workflows/build-osx.yml index 56ab0809..a2d508c0 100644 --- a/.github/workflows/build-osx.yml +++ b/.github/workflows/build-osx.yml @@ -31,7 +31,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: Debug build - path: ${{github.workspace}}/build/chia_plot + path: | + ${{github.workspace}}/build/chia_plot + ${{github.workspace}}/build/chia_plot_k34 build-release: runs-on: macOS-latest @@ -52,7 +54,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: Release build - path: ${{github.workspace}}/build/chia_plot + path: | + ${{github.workspace}}/build/chia_plot + ${{github.workspace}}/build/chia_plot_k34 - name: Get tag name if: startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 8c925339..41c23d1b 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -37,7 +37,9 @@ jobs: uses: actions/upload-artifact@v2.2.3 with: name: MSVC Release build - path: ${{github.workspace}}/build/chia_plot.exe + path: | + ${{github.workspace}}/build/chia_plot.exe + ${{github.workspace}}/build/chia_plot_k34.exe upload-release-artifacts: runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index a72dacf1..860c1fba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,6 +161,24 @@ target_link_libraries(chia_plot bls ) +add_executable(chia_plot_k34 src/chia_plot.cpp) + +target_include_directories(chia_plot_k34 + PRIVATE + ${libbech32_SOURCE_DIR}/include/libbech32 +) + +target_compile_definitions(chia_plot_k34 + PUBLIC + CHIA_K34=1 +) + +target_link_libraries(chia_plot_k34 + PRIVATE + chia_plotter + bls +) + if(CHIA_PLOTTER_BUILD_TESTS) add_executable(test_copy test/test_copy.cpp) target_link_libraries(test_copy PRIVATE chia_plotter) diff --git a/README.md b/README.md index b1c3d33d..e73302f2 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ Usage: -c, --contract arg Pool Contract Address (62 chars) -f, --farmerkey arg Farmer Public Key (48 bytes) -G, --tmptoggle Alternate tmpdir/tmpdir2 (default = false) + -D, --directout Create plot directly in finaldir (default = false) -K, --rmulti2 arg Thread multiplier for P2 (default = 1) --help Print help ``` diff --git a/include/chia/DiskSort.hpp b/include/chia/DiskSort.hpp index 3f5e6722..fdbde4ef 100644 --- a/include/chia/DiskSort.hpp +++ b/include/chia/DiskSort.hpp @@ -24,7 +24,7 @@ void DiskSort::bucket_t::open(const char* mode) } file = FOPEN(file_name.c_str(), mode); if(!file) { - throw std::runtime_error("fopen() failed"); + throw std::runtime_error("fopen() failed with: " + std::string(std::strerror(errno))); } } @@ -34,7 +34,7 @@ void DiskSort::bucket_t::write(const void* data, size_t count) std::lock_guard lock(mutex); if(file) { if(fwrite(data, T::disk_size, count, file) != count) { - throw std::runtime_error("fwrite() failed"); + throw std::runtime_error("fwrite() failed with: " + std::string(std::strerror(errno))); } num_entries += count; } @@ -44,7 +44,9 @@ template void DiskSort::bucket_t::close() { if(file) { - fclose(file); + if(fclose(file)) { + throw std::runtime_error("fclose() failed with: " + std::string(std::strerror(errno))); + } file = nullptr; } } @@ -67,7 +69,8 @@ void DiskSort::WriteCache::add(const T& entry) { const size_t index = Key{}(entry) >> key_shift; if(index >= buckets.size()) { - throw std::logic_error("bucket index out of range"); + throw std::logic_error("bucket index out of range: " + + std::to_string(index) + " >= " + std::to_string(buckets.size())); } auto& buffer = buckets[index]; if(buffer.count >= buffer.capacity) { @@ -197,7 +200,7 @@ void DiskSort::read_bucket( std::pair& index, { const size_t num_entries = std::min(buffer.capacity, bucket.num_entries - i); if(fread(buffer.data, T::disk_size, num_entries, bucket.file) != num_entries) { - throw std::runtime_error("fread() failed"); + throw std::runtime_error("fread() failed with: " + std::string(std::strerror(errno))); } for(size_t k = 0; k < num_entries; ++k) { T entry; diff --git a/include/chia/DiskTable.h b/include/chia/DiskTable.h index 2c7088f4..63b8686c 100644 --- a/include/chia/DiskTable.h +++ b/include/chia/DiskTable.h @@ -36,6 +36,9 @@ class DiskTable { { if(!num_entries) { file_out = FOPEN(file_name.c_str(), "wb"); + if(!file_out) { + throw std::runtime_error("fopen() failed with: " + std::string(std::strerror(errno))); + } } } @@ -71,7 +74,7 @@ class DiskTable { { FILE* file = FOPEN(file_name.c_str(), "rb"); if(!file) { - throw std::runtime_error("fopen() failed"); + throw std::runtime_error("fopen() failed with: " + std::string(std::strerror(errno))); } auto& local = pool.get_local(i); local.file = file; @@ -101,8 +104,11 @@ class DiskTable { } void flush() { + if(!file_out) { + throw std::logic_error("read only"); + } if(fwrite(cache.data, cache.entry_size, cache.count, file_out) != cache.count) { - throw std::runtime_error("fwrite() failed"); + throw std::runtime_error("fwrite() failed with: " + std::string(std::strerror(errno))); } num_entries += cache.count; cache.count = 0; @@ -122,10 +128,10 @@ class DiskTable { local_t& local) const { if(int err = FSEEK(local.file, param.first * T::disk_size, SEEK_SET)) { - throw std::runtime_error("fseek() failed"); + throw std::runtime_error("fseek() failed with: " + std::string(std::strerror(errno))); } if(fread(local.buffer, T::disk_size, param.second, local.file) != param.second) { - throw std::runtime_error("fread() failed"); + throw std::runtime_error("fread() failed with: " + std::string(std::strerror(errno))); } auto& entries = out.first; entries.resize(param.second); diff --git a/include/chia/bits.hpp b/include/chia/bits.hpp index b17b30da..195ea262 100644 --- a/include/chia/bits.hpp +++ b/include/chia/bits.hpp @@ -554,4 +554,61 @@ using Bits = BitsGeneric; using ParkBits = BitsGeneric; using LargeBits = BitsGeneric; + +inline +int write_bits(uint64_t* dst, const uint64_t value, const int bit_offset, const int num_bits) +{ + assert(num_bits <= 64); + const int free_bits = 64 - (bit_offset % 64); + if(free_bits >= num_bits) { + dst[bit_offset / 64] |= bswap_64(value << (free_bits - num_bits)); + } else { + const int suffix_size = num_bits - free_bits; + const uint64_t suffix = value & ((uint64_t(1) << suffix_size) - 1); + dst[bit_offset / 64] |= bswap_64(value >> suffix_size); // prefix (high bits) + dst[bit_offset / 64 + 1] |= bswap_64(suffix << (64 - suffix_size)); // suffix (low bits) + } + return bit_offset + num_bits; +} + +inline +int append_bits(uint64_t* dst, const uint64_t* src, const int bit_offset, const int num_bits) +{ + int i = 0; + int offset = bit_offset; + int num_left = num_bits; + while(num_left > 0) { + int bits = 64; + uint64_t value = bswap_64(src[i]); + if(num_left < 64) { + bits = num_left; + value >>= (64 - num_left); + } + offset = write_bits(dst, value, offset, bits); + num_left -= bits; + i++; + } + return offset; +} + +inline +int slice_bits(uint64_t* dst, const uint64_t* src, const int bit_offset, const int num_bits) +{ + int count = 0; + int offset = bit_offset; + int num_left = num_bits; + while(num_left > 0) { + const int shift = offset % 64; + const int bits = std::min(num_left, 64 - shift); + uint64_t value = bswap_64(src[offset / 64]) << shift; + if(bits < 64) { + value >>= (64 - bits); + } + count = write_bits(dst, value, count, bits); + offset += bits; + num_left -= bits; + } + return count; +} + #endif // SRC_CPP_BITS_HPP_ diff --git a/include/chia/copy.h b/include/chia/copy.h index 58b7e8f1..ccab2d19 100644 --- a/include/chia/copy.h +++ b/include/chia/copy.h @@ -14,6 +14,8 @@ #include #include +#include +#include #ifdef _MSC_VER #include @@ -27,21 +29,23 @@ uint64_t copy_file(const std::string& src_path, const std::string& dst_path) { FILE* src = FOPEN(src_path.c_str(), "rb"); if(!src) { - throw std::runtime_error("fopen() failed for " + src_path); + throw std::runtime_error("fopen() failed for " + src_path + " (" + std::string(std::strerror(errno)) + ")"); } FILE* dst = FOPEN(dst_path.c_str(), "wb"); if(!dst) { + const auto err = errno; fclose(src); - throw std::runtime_error("fopen() failed for " + dst_path); + throw std::runtime_error("fopen() failed for " + dst_path + " (" + std::string(std::strerror(err)) + ")"); } uint64_t total_bytes = 0; std::vector buffer(g_read_chunk_size * 16); while(true) { const auto num_bytes = fread(buffer.data(), 1, buffer.size(), src); if(fwrite(buffer.data(), 1, num_bytes, dst) != num_bytes) { + const auto err = errno; fclose(src); fclose(dst); - throw std::runtime_error("fwrite() failed on " + dst_path); + throw std::runtime_error("fwrite() failed on " + dst_path + " (" + std::string(std::strerror(err)) + ")"); } total_bytes += num_bytes; if(num_bytes < buffer.size()) { @@ -51,7 +55,7 @@ uint64_t copy_file(const std::string& src_path, const std::string& dst_path) fclose(src); if(fclose(dst)) { - throw std::runtime_error("fclose() failed on " + dst_path); + throw std::runtime_error("fclose() failed on " + dst_path + " (" + std::string(std::strerror(errno)) + ")"); } return total_bytes; } diff --git a/include/chia/entries.h b/include/chia/entries.h index 1e51449d..1361ea3e 100644 --- a/include/chia/entries.h +++ b/include/chia/entries.h @@ -8,9 +8,26 @@ #ifndef INCLUDE_CHIA_ENTRIES_H_ #define INCLUDE_CHIA_ENTRIES_H_ +#include + #include #include +#ifdef CHIA_K34 + constexpr int KMAX = 34; + constexpr int PMAX = 35; + constexpr int KBYTES = 5; + typedef uint64_t uintkx_t; + typedef uint128_t uintlp_t; +#else +#define CHIA_K32 + constexpr int KMAX = 32; + constexpr int PMAX = 32; + constexpr int KBYTES = 4; + typedef uint32_t uintkx_t; + typedef uint64_t uintlp_t; +#endif + template bool write_entry(FILE* file, const T& entry) { diff --git a/include/chia/phase1.h b/include/chia/phase1.h index dff9cdfa..8e9c623c 100644 --- a/include/chia/phase1.h +++ b/include/chia/phase1.h @@ -30,38 +30,65 @@ struct input_t { }; struct entry_1 { - uint64_t y; // 38 bit - uint32_t x; // 32 bit + uint64_t y; // 38 bit / 40 bit + uintkx_t x; // 32 bit / 34 bit - static constexpr uint32_t pos = 0; // dummy + static constexpr uintkx_t pos = 0; // dummy static constexpr uint16_t off = 0; // dummy - static constexpr size_t disk_size = 9; + static constexpr size_t disk_size = 5 + KBYTES; size_t read(const uint8_t* buf) { y = 0; memcpy(&y, buf, 5); - memcpy(&x, buf + 5, 4); + if(sizeof(x) > KBYTES) { + x = 0; + } + memcpy(&x, buf + 5, KBYTES); // 32 bit / 40 bit return disk_size; } size_t write(uint8_t* buf) const { memcpy(buf, &y, 5); - memcpy(buf + 5, &x, 4); + memcpy(buf + 5, &x, KBYTES); return disk_size; } }; struct entry_x { - uint64_t y; // 38 bit - uint32_t pos; // 32 bit + uint64_t y; // 38 bit / 40 bit + uintkx_t pos; // 32 bit / 35 bit uint16_t off; // 10 bit }; template struct entry_xm : entry_x { - std::array meta; - - static constexpr size_t disk_size = 10 + N * 4; + std::array meta; + +#ifdef CHIA_K34 + static constexpr size_t disk_size = 11 + N; + size_t read(const uint8_t* buf) { + y = 0; + memcpy(&y, buf, 5); + memcpy(&pos, buf + 5, 5); + pos &= 0x3FFFFFFFFF; // 38 bit + memcpy(&off, buf + 9, 2); + off >>= 6; + memcpy(meta.data(), buf + 11, meta.size()); + return disk_size; + } + size_t write(uint8_t* buf) const { + memcpy(buf, &y, 5); + memcpy(buf + 5, &pos, 5); + { + const auto tmp = (off << 6) | buf[9]; + memcpy(buf + 9, &tmp, 2); + } + memcpy(buf + 11, meta.data(), meta.size()); + return disk_size; + } +#else + static constexpr size_t disk_size = 10 + N; + size_t read(const uint8_t* buf) { memcpy(&y, buf, 5); y &= 0x3FFFFFFFFFull; @@ -69,7 +96,7 @@ struct entry_xm : entry_x { off |= buf[4] >> 6; off |= uint16_t(buf[5]) << 2; memcpy(&pos, buf + 6, 4); - memcpy(meta.data(), buf + 10, sizeof(meta)); + memcpy(meta.data(), buf + 10, meta.size()); return disk_size; } size_t write(uint8_t* buf) const { @@ -77,22 +104,54 @@ struct entry_xm : entry_x { buf[4] = (off << 6) | (buf[4] & 0x3F); buf[5] = off >> 2; memcpy(buf + 6, &pos, 4); - memcpy(buf + 10, meta.data(), sizeof(meta)); + memcpy(buf + 10, meta.data(), meta.size()); return disk_size; } +#endif }; -typedef entry_xm<2> entry_2; -typedef entry_xm<4> entry_3; -typedef entry_xm<4> entry_4; -typedef entry_xm<3> entry_5; -typedef entry_xm<2> entry_6; +#ifdef CHIA_K34 +typedef entry_xm<9> entry_2; +typedef entry_xm<17> entry_3; +typedef entry_xm<17> entry_4; +typedef entry_xm<13> entry_5; +typedef entry_xm<9> entry_6; +#else +typedef entry_xm<8> entry_2; +typedef entry_xm<16> entry_3; +typedef entry_xm<16> entry_4; +typedef entry_xm<12> entry_5; +typedef entry_xm<8> entry_6; +#endif struct entry_7 { - uint32_t y; // 32 bit - uint32_t pos; // 32 bit + uintkx_t y; // 32 bit / 34 bit + uintkx_t pos; // 32 bit / 35 bit uint16_t off; // 10 bit +#ifdef CHIA_K34 + static constexpr size_t disk_size = 11; + + void assign(const entry_7& entry) { + *this = entry; + } + size_t read(const uint8_t* buf) { + memcpy(&y, buf, 5); + y &= 0xFFFFFFFFF; // 36 bit + memcpy(&pos, buf + 4, 5); + pos >>= 4; + pos &= 0xFFFFFFFFF; // 36 bit + memcpy(&off, buf + 9, 2); + return disk_size; + } + size_t write(uint8_t* buf) const { + memcpy(buf, &y, 5); + const auto tmp = (pos << 4) | buf[4]; + memcpy(buf + 4, &tmp, 5); + memcpy(buf + 9, &off, 2); + return disk_size; + } +#else static constexpr size_t disk_size = 10; void assign(const entry_7& entry) { @@ -110,28 +169,32 @@ struct entry_7 { memcpy(buf + 8, &off, 2); return disk_size; } +#endif }; struct tmp_entry_1 { - uint32_t x; // 32 bit + uintkx_t x; // 32 bit / 34 bit - static constexpr size_t disk_size = 4; + static constexpr size_t disk_size = KBYTES; void assign(const entry_1& entry) { x = entry.x; } size_t read(const uint8_t* buf) { - memcpy(&x, buf, 4); + if(sizeof(x) > KBYTES) { + x = 0; + } + memcpy(&x, buf, KBYTES); return disk_size; } size_t write(uint8_t* buf) const { - memcpy(buf, &x, 4); + memcpy(buf, &x, KBYTES); return disk_size; } }; struct tmp_entry_x { - uint32_t pos; // 32 bit + uintkx_t pos; // 32 bit / 35 bit uint16_t off; // 10 bit static constexpr size_t disk_size = 6; @@ -141,13 +204,19 @@ struct tmp_entry_x { off = entry.off; } size_t read(const uint8_t* buf) { - memcpy(&pos, buf, 4); + memcpy(&pos, buf, KBYTES); + pos &= 0x3FFFFFFFFF; // 38 bit memcpy(&off, buf + 4, 2); + off >>= 6; return disk_size; } size_t write(uint8_t* buf) const { - memcpy(buf, &pos, 4); - memcpy(buf + 4, &off, 2); + memcpy(buf, &pos, KBYTES); + if(KBYTES < 5) { + buf[4] = 0; + } + const auto tmp = (off << 6) | buf[4]; + memcpy(buf + 4, &tmp, 2); return disk_size; } }; @@ -161,30 +230,26 @@ struct get_y { template struct get_meta { - void operator()(const T& entry, uint128_t* value) { - *value = 0; - memcpy(value, entry.meta.data(), sizeof(entry.meta)); + void operator()(const T& entry, uint64_t* bytes, const int k) { + memcpy(bytes, entry.meta.data(), entry.meta.size()); } }; template<> struct get_meta { - void operator()(const entry_1& entry, uint128_t* value) { - *value = entry.x; - } + void operator()(const entry_1& entry, uint64_t* bytes, const int k); }; template struct set_meta { - void operator()(T& entry, const uint128_t value, const size_t num_bytes) { - entry.meta = {}; - memcpy(entry.meta.data(), &value, num_bytes); + void operator()(T& entry, const uint64_t* bytes, const size_t num_bytes) { + memcpy(entry.meta.data(), bytes, num_bytes); } }; template<> struct set_meta { - void operator()(entry_7& entry, const uint128_t value, const size_t num_bytes) { + void operator()(entry_7& entry, const uint64_t* bytes, const size_t num_bytes) { // no meta data } }; @@ -193,7 +258,7 @@ template struct match_t { T left; T right; - uint32_t pos = 0; + uintkx_t pos = 0; uint16_t off = 0; }; diff --git a/include/chia/phase1.hpp b/include/chia/phase1.hpp index a79198dd..c2d0db11 100644 --- a/include/chia/phase1.hpp +++ b/include/chia/phase1.hpp @@ -39,6 +39,10 @@ static void initialize() { load_tables(); } +void get_meta::operator()(const entry_1& entry, uint64_t* bytes, const int k) { + write_bits(bytes, entry.x, 0, k); +} + class F1Calculator { public: F1Calculator(int k, const uint8_t* orig_key) @@ -62,12 +66,12 @@ class F1Calculator { const uint64_t num_blocks = end - start; const uint8_t x_shift = k_ - kExtraBits; - if(num_blocks > 2) { - throw std::logic_error("num_blocks > 2"); + if(num_blocks > 4) { + throw std::logic_error("num_blocks > 4"); } uint32_t start_bit = (first_x * k_) % kF1BlockSizeBits; - uint8_t buf[2 * 64]; + uint8_t buf[4 * 64]; chacha8_get_keystream(&this->enc_ctx_, start, num_blocks, buf); for(uint64_t x = first_x; x < first_x + num_entries; x++) @@ -101,53 +105,44 @@ class FxCalculator { // Performs one evaluation of the f function. void evaluate(const T& L, const T& R, S& entry) const { - Bits C; - Bits input; - uint8_t input_bytes[64]; - uint8_t hash_bytes[32]; - uint128_t L_meta; - uint128_t R_meta; + uint64_t C[4] = {}; + uint64_t input[8] = {}; + uint64_t L_meta[4] = {}; + uint64_t R_meta[4] = {}; + uint64_t hash_bytes[4]; + + size_t C_bits = 0; + size_t input_bits = 0; const int meta_bits = kVectorLens[table_index_] * k_; - get_meta{}(L, &L_meta); - get_meta{}(R, &R_meta); - - const Bits Y_1(L.y, k_ + kExtraBits); - const Bits L_c(L_meta, meta_bits); - const Bits R_c(R_meta, meta_bits); + get_meta{}(L, L_meta, k_); + get_meta{}(R, R_meta, k_); if (table_index_ < 4) { - C = L_c + R_c; - input = Y_1 + C; + C_bits = append_bits(C, L_meta, C_bits, meta_bits); + C_bits = append_bits(C, R_meta, C_bits, meta_bits); + input_bits = write_bits(input, L.y, input_bits, k_ + kExtraBits); + input_bits = append_bits(input, C, input_bits, C_bits); } else { - input = Y_1 + L_c + R_c; + input_bits = write_bits(input, L.y, input_bits, k_ + kExtraBits); + input_bits = append_bits(input, L_meta, input_bits, meta_bits); + input_bits = append_bits(input, R_meta, input_bits, meta_bits); } - input.ToBytes(input_bytes); blake3_hasher hasher; blake3_hasher_init(&hasher); - blake3_hasher_update(&hasher, input_bytes, cdiv(input.GetSize(), 8)); - blake3_hasher_finalize(&hasher, hash_bytes, sizeof(hash_bytes)); + blake3_hasher_update(&hasher, input, cdiv(input_bits, 8)); + blake3_hasher_finalize(&hasher, (uint8_t*)hash_bytes, 32); - entry.y = Util::EightBytesToInt(hash_bytes) >> (64 - (k_ + (table_index_ < 7 ? kExtraBits : 0))); + entry.y = bswap_64(hash_bytes[0]) >> (64 - (k_ + (table_index_ < 7 ? kExtraBits : 0))); if (table_index_ < 4) { // c is already computed } else if (table_index_ < 7) { - uint8_t len = kVectorLens[table_index_ + 1]; - uint8_t start_byte = (k_ + kExtraBits) / 8; - uint8_t end_bit = k_ + kExtraBits + k_ * len; - uint8_t end_byte = cdiv(end_bit, 8); - - // TODO: proper support for partial bytes in Bits ctor - C = Bits(hash_bytes + start_byte, end_byte - start_byte, (end_byte - start_byte) * 8); - - C = C.Slice((k_ + kExtraBits) % 8, end_bit - start_byte * 8); + C_bits = slice_bits(C, hash_bytes, k_ + kExtraBits, kVectorLens[table_index_ + 1] * k_); } - uint8_t C_bytes[16]; - C.ToBytes(C_bytes); - set_meta{}(entry, Util::SliceInt128FromBytes(C_bytes, 0, C.GetSize()), cdiv(C.GetSize(), 8)); + set_meta{}(entry, C, cdiv(C_bits, 8)); } private: @@ -203,11 +198,12 @@ class FxMatcher { for (size_t pos_R = 0; pos_R < bucket_R.size(); pos_R++) { const uint64_t r_y = bucket_R[pos_R].y - offset; - if (!rmap[r_y].count) { - rmap[r_y].pos = pos_R; + auto& entry = rmap[r_y]; + if (!entry.count) { + entry.pos = pos_R; rmap_clean.push_back(r_y); } - rmap[r_y].count++; + entry.count++; } int idx_count = 0; @@ -235,16 +231,20 @@ class FxMatcher { uint16_t idx_R[kBC]; const int count = find_matches_ex(bucket_L, bucket_R, idx_L, idx_R); + if(count > kBC) { + throw std::logic_error("find_matches(): count > kBC"); + } for(int i = 0; i < count; ++i) { const auto pos = L_pos_begin + idx_L[i]; - if(pos < (uint64_t(1) << 32)) { - match_t match; - match.left = bucket_L[idx_L[i]]; - match.right = bucket_R[idx_R[i]]; - match.pos = pos; - match.off = idx_R[i] + (bucket_L.size() - idx_L[i]); - out.push_back(match); + if(pos >= (uint64_t(1) << PMAX)) { + continue; } + match_t match; + match.left = bucket_L[idx_L[i]]; + match.right = bucket_R[idx_R[i]]; + match.pos = pos; + match.off = idx_R[i] + (bucket_L.size() - idx_L[i]); + out.push_back(match); } return count; } @@ -412,7 +412,7 @@ uint64_t compute_matches( int R_index, int k, int num_threads, } if(num_written < num_found) { // std::cout << "[P1] Lost " << num_found - num_written -// << " matches due to 32-bit overflow." << std::endl; +// << " matches due to PMAX-bit overflow." << std::endl; } return num_written; } diff --git a/include/chia/phase2.h b/include/chia/phase2.h index cbc938fc..dddfb881 100644 --- a/include/chia/phase2.h +++ b/include/chia/phase2.h @@ -23,16 +23,40 @@ namespace phase2 { struct entry_x { - uint32_t key; - uint32_t pos; + uintkx_t key; // 32 bit / 35 bit + uintkx_t pos; // 32 bit / 35 bit uint16_t off; // 10 bit static constexpr size_t disk_size = 10; - + void assign(const phase1::tmp_entry_x& entry) { pos = entry.pos; off = entry.off; } +#ifdef CHIA_K34 + size_t read(const uint8_t* buf) { + memcpy(&key, buf, 5); + key &= 0x7FFFFFFFF; // 35 bit + memcpy(&pos, buf + 4, 5); + pos >>= 3; + pos &= 0x7FFFFFFFF; // 35 bit + memcpy(&off, buf + 8, 2); + off >>= 6; + return disk_size; + } + size_t write(uint8_t* buf) const { + memcpy(buf, &key, 5); + { + const auto tmp = (pos << 3) | buf[4]; + memcpy(buf + 4, &tmp, 5); + } + { + const auto tmp = (off << 6) | buf[8]; + memcpy(buf + 8, &tmp, 2); + } + return disk_size; + } +#else size_t read(const uint8_t* buf) { memcpy(&key, buf, 4); memcpy(&pos, buf + 4, 4); @@ -45,6 +69,7 @@ struct entry_x { memcpy(buf + 8, &off, 2); return disk_size; } +#endif }; typedef phase1::tmp_entry_1 entry_1; @@ -59,14 +84,14 @@ struct get_pos { template struct set_sort_key { - void operator()(T& entry, uint32_t key) { + void operator()(T& entry, uint64_t key) { entry.key = key; } }; template<> struct set_sort_key { - void operator()(entry_7& entry, uint32_t key) { + void operator()(entry_7& entry, uint64_t key) { // no sort key } }; diff --git a/include/chia/phase3.h b/include/chia/phase3.h index f0662d95..2604832d 100644 --- a/include/chia/phase3.h +++ b/include/chia/phase3.h @@ -14,14 +14,32 @@ namespace phase3 { struct entry_kpp { - uint32_t pos[2]; // 2x 32-bit position - uint32_t key; // 32-bit (sort_key) + uintkx_t pos[2]; // 2x 32-bit position / 2x 35-bit + uintkx_t key; // 32-bit (sort_key) / 35 bit }; struct entry_lp { - uint64_t point; // 63-bit (line_point) - uint32_t key; // 32-bit (sort_key) + uintlp_t point; // 63-bit (line_point) / 67 bit + uintkx_t key; // 32-bit (sort_key) / 35 bit +#ifdef CHIA_K34 + static constexpr size_t disk_size = 13; + + size_t read(const uint8_t* buf) { + memcpy(&point, buf, 9); + point &= (uint128_t(1) << 68) - 1; // 68 bit + key = 0; + memcpy(&key, buf + 8, 5); + key >>= 4; // 36 bit + return disk_size; + } + size_t write(uint8_t* buf) const { + memcpy(buf, &point, 9); + const auto tmp = (key << 4) | buf[8]; + memcpy(buf + 8, &tmp, 5); + return disk_size; + } +#else static constexpr size_t disk_size = 12; size_t read(const uint8_t* buf) { @@ -34,12 +52,31 @@ struct entry_lp { memcpy(buf + 8, &key, 4); return disk_size; } +#endif }; struct entry_np { - uint32_t key; // 32-bit (sort_key) - uint32_t pos; // 32-bit (new_pos) + uintkx_t key; // 32-bit (sort_key) / 35 bit + uintkx_t pos; // 32-bit (new_pos) / 35 bit +#ifdef CHIA_K34 + static constexpr size_t disk_size = 9; + + size_t read(const uint8_t* buf) { + memcpy(&key, buf, 5); + key &= 0xFFFFFFFFF; // 36 bit + pos = 0; + memcpy(&pos, buf + 4, 5); + pos >>= 4; // 36 bit + return disk_size; + } + size_t write(uint8_t* buf) const { + memcpy(buf, &key, 5); + const auto tmp = (pos << 4) | buf[4]; + memcpy(buf + 4, &tmp, 5); + return disk_size; + } +#else static constexpr size_t disk_size = 8; size_t read(const uint8_t* buf) { @@ -52,39 +89,40 @@ struct entry_np { memcpy(buf + 4, &pos, 4); return disk_size; } +#endif }; template struct get_new_pos { - uint32_t operator()(const T& entry) { + uint64_t operator()(const T& entry) { return entry.pos; } }; template<> struct get_new_pos { - uint32_t operator()(const phase2::entry_1& entry) { + uint64_t operator()(const phase2::entry_1& entry) { return entry.x; } }; template struct get_sort_key { - uint32_t operator()(const T& entry) { + uint64_t operator()(const T& entry) { return entry.key; } }; template<> struct get_sort_key { - uint32_t operator()(const phase2::entry_7& entry) { + uint64_t operator()(const phase2::entry_7& entry) { return entry.y; } }; template struct get_line_point { - uint64_t operator()(const T& entry) { + uint128_t operator()(const T& entry) { return entry.point; } }; diff --git a/include/chia/phase3.hpp b/include/chia/phase3.hpp index d6aabcbc..50079735 100644 --- a/include/chia/phase3.hpp +++ b/include/chia/phase3.hpp @@ -29,7 +29,7 @@ void compute_stage1(int L_index, int num_threads, struct merge_buffer_t { uint64_t offset = 0; // position offset at buffer[0] - std::vector new_pos; // new_pos buffer + std::vector new_pos; // new_pos buffer int copy_sync = 0; // copy counter }; @@ -339,7 +339,7 @@ uint64_t compute_stage2(int L_index, int k, int num_threads, struct park_data_t { uint64_t index = 0; - std::vector points; + std::vector points; } park; struct park_out_t { @@ -359,8 +359,8 @@ uint64_t compute_stage2(int L_index, int k, int num_threads, } uint64_t index = input.second; for(const auto& entry : input.first) { - if(index >= uint64_t(1) << 32) { - break; // skip 32-bit overflow + if(index >= uint64_t(1) << PMAX) { + break; // skip PMAX-bit overflow } entry_np tmp; tmp.key = entry.key; @@ -392,7 +392,7 @@ uint64_t compute_stage2(int L_index, int k, int num_threads, const auto stub = big_delta & ((1ull << (k - kStubMinusBits)) - 1); const auto small_delta = big_delta >> (k - kStubMinusBits); if(small_delta >= 256) { - throw std::logic_error("small_delta >= 256 (" + std::to_string(small_delta) + ")"); + throw std::logic_error("small_delta >= 256 (" + std::to_string(uint64_t(small_delta)) + ")"); } deltas[i] = small_delta; stubs[i] = stub; @@ -419,8 +419,8 @@ uint64_t compute_stage2(int L_index, int k, int num_threads, parks.reserve(input.first.size() / kEntriesPerPark + 2); uint64_t index = input.second; for(const auto& entry : input.first) { - if(index >= uint64_t(1) << 32) { - break; // skip 32-bit overflow + if(index >= uint64_t(1) << PMAX) { + break; // skip PMAX-bit overflow } // Every EPP entries, writes a park if(index % kEntriesPerPark == 0) { @@ -460,7 +460,7 @@ uint64_t compute_stage2(int L_index, int k, int num_threads, Encoding::ANSFree(kRValues[L_index - 1]); if(L_num_write < R_num_read) { -// std::cout << "[P3-2] Lost " << R_num_read - L_num_write << " entries due to 32-bit overflow." << std::endl; +// std::cout << "[P3-2] Lost " << R_num_read - L_num_write << " entries due to PMAX-bit overflow." << std::endl; } std::cout << "[P3-2] Table " << L_index + 1 << " took " << (get_wall_time_micros() - begin) / 1e6 << " sec" @@ -474,7 +474,8 @@ void compute( phase2::output_t& input, output_t& out, const int num_threads, const int log_num_buckets, const std::string plot_name, const std::string tmp_dir, - const std::string tmp_dir_2) + const std::string tmp_dir_2, + const std::string plot_dir) { const auto total_begin = get_wall_time_micros(); @@ -482,11 +483,11 @@ void compute( phase2::output_t& input, output_t& out, const std::string prefix_2 = tmp_dir_2 + plot_name + "."; out.params = input.params; - out.plot_file_name = tmp_dir + plot_name + ".plot.tmp"; + out.plot_file_name = plot_dir + plot_name + ".plot.tmp"; FILE* plot_file = FOPEN(out.plot_file_name.c_str(), "wb"); if(!plot_file) { - throw std::runtime_error("fopen() failed"); + throw std::runtime_error("fopen() failed with: " + std::string(std::strerror(errno))); } out.header_size = WriteHeader( plot_file, k, input.params.id.data(), input.params.memo.data(), input.params.memo.size()); @@ -554,7 +555,10 @@ void compute( phase2::output_t& input, output_t& out, Util::IntToEightBytes(tmp, final_pointers[i]); fwrite_ex(plot_file, tmp, sizeof(tmp)); } - fclose(plot_file); + if(fclose(plot_file)) { + throw std::runtime_error("fclose() failed with: " + std::string(std::strerror(errno))); + } + plot_file = nullptr; out.sort_7 = L_sort_np; out.num_written_7 = num_written_final_7; diff --git a/include/chia/phase4.hpp b/include/chia/phase4.hpp index 49ba63ed..b90cbb80 100644 --- a/include/chia/phase4.hpp +++ b/include/chia/phase4.hpp @@ -76,7 +76,7 @@ uint64_t compute( FILE* plot_file, uint64_t prev_y = 0; uint64_t num_C1_entries = 0; - std::vector C2; + std::vector C2; std::cout << "[P4] Starting to write C1 and C3 tables" << std::endl; @@ -87,7 +87,7 @@ uint64_t compute( FILE* plot_file, struct park_data_t { uint64_t offset = 0; - std::vector array; // new_pos + std::vector array; // new_pos } park_data; struct write_data_t { @@ -209,7 +209,7 @@ uint64_t compute( FILE* plot_file, p7_threads.close(); plot_write.close(); - uint8_t C1_entry_buf[4] = {}; + uint8_t C1_entry_buf[8] = {}; Bits(0, Util::ByteAlign(k)).ToBytes(C1_entry_buf); final_file_writer_1 += fwrite_at(plot_file, final_file_writer_1, C1_entry_buf, Util::ByteAlign(k) / 8); @@ -217,7 +217,7 @@ uint64_t compute( FILE* plot_file, std::cout << "[P4] Finished writing C1 and C3 tables" << std::endl; std::cout << "[P4] Writing C2 table" << std::endl; - for(const uint64_t C2_entry : C2) { + for(auto C2_entry : C2) { Bits(C2_entry, k).ToBytes(C1_entry_buf); final_file_writer_1 += fwrite_at(plot_file, final_file_writer_1, C1_entry_buf, Util::ByteAlign(k) / 8); @@ -245,7 +245,8 @@ void compute( const phase3::output_t& input, output_t& out, const int num_threads, const int log_num_buckets, const std::string plot_name, const std::string tmp_dir, - const std::string tmp_dir_2) + const std::string tmp_dir_2, + const std::string plot_dir) { const auto total_begin = get_wall_time_micros(); @@ -260,7 +261,7 @@ void compute( const phase3::output_t& input, output_t& out, fclose(plot_file); out.params = input.params; - out.plot_file_name = tmp_dir + plot_name + ".plot"; + out.plot_file_name = plot_dir + plot_name + ".plot"; std::rename(input.plot_file_name.c_str(), out.plot_file_name.c_str()); diff --git a/src/chia_plot.cpp b/src/chia_plot.cpp index 66c7d47c..e55622f3 100644 --- a/src/chia_plot.cpp +++ b/src/chia_plot.cpp @@ -88,7 +88,8 @@ phase4::output_t create_plot( const int k, const vector& puzzle_hash_bytes, const vector& farmer_key_bytes, const std::string& tmp_dir, - const std::string& tmp_dir_2) + const std::string& tmp_dir_2, + const std::string& plot_dir) { const auto total_begin = get_wall_time_micros(); const bool have_puzzle = !puzzle_hash_bytes.empty(); @@ -192,10 +193,10 @@ phase4::output_t create_plot( const int k, phase2::compute(out_1, out_2, num_threads, log_num_buckets_3, plot_name, tmp_dir, tmp_dir_2); phase3::output_t out_3; - phase3::compute(out_2, out_3, num_threads, log_num_buckets_3, plot_name, tmp_dir, tmp_dir_2); + phase3::compute(out_2, out_3, num_threads, log_num_buckets_3, plot_name, tmp_dir, tmp_dir_2, plot_dir); phase4::output_t out_4; - phase4::compute(out_3, out_4, num_threads, log_num_buckets_3, plot_name, tmp_dir, tmp_dir_2); + phase4::compute(out_3, out_4, num_threads, log_num_buckets_3, plot_name, tmp_dir, tmp_dir_2, plot_dir); const auto time_secs = (get_wall_time_micros() - total_begin) / 1e6; std::cout << "Total plot creation time was " @@ -208,7 +209,7 @@ int _main(int argc, char** argv) { cxxopts::Options options("chia_plot", - "Multi-threaded pipelined Chia k32 plotter" + "Multi-threaded pipelined Chia k" + std::to_string(KMAX) + " plotter" #ifdef GIT_COMMIT_HASH " - " GIT_COMMIT_HASH #endif @@ -236,9 +237,10 @@ int _main(int argc, char** argv) int num_buckets_3 = 0; bool waitforcopy = false; bool tmptoggle = false; + bool directout = false; options.allow_unrecognised_options().add_options()( - "k, size", "K size (default = 32, k <= 32)", cxxopts::value(k))( + "k, size", "K size (default = 32, k <= " + std::to_string(KMAX) + ")", cxxopts::value(k))( "x, port", "Network port (default = 8444, chives = 9699)", cxxopts::value(port))( "n, count", "Number of plots to create (default = 1, -1 = infinite)", cxxopts::value(num_plots))( "r, threads", "Number of threads (default = 4)", cxxopts::value(num_threads))( @@ -251,7 +253,8 @@ int _main(int argc, char** argv) "p, poolkey", "Pool Public Key (48 bytes)", cxxopts::value(pool_key_str))( "c, contract", "Pool Contract Address (62 chars)", cxxopts::value(contract_addr_str))( "f, farmerkey", "Farmer Public Key (48 bytes)", cxxopts::value(farmer_key_str))( - "G, tmptoggle", "Alternate tmpdir/tmpdir2", cxxopts::value(tmptoggle))( + "G, tmptoggle", "Alternate tmpdir/tmpdir2 (default = false)", cxxopts::value(tmptoggle))( + "D, directout", "Create plot directly in finaldir (default = false)", cxxopts::value(directout))( "K, rmulti2", "Thread multiplier for P2 (default = 1)", cxxopts::value(phase2::g_thread_multi))( "version", "Print version")( "help", "Print help"); @@ -270,7 +273,7 @@ int _main(int argc, char** argv) std::cout << kVersion << std::endl; return 0; } - if(k > 32 || k < 16) { + if(k > KMAX || k < 16) { std::cout << "Invalid k option: " << k << std::endl; return -2; } @@ -444,7 +447,7 @@ int _main(int argc, char** argv) std::cout << std::endl; } - std::cout << "Multi-threaded pipelined Chia k32 plotter"; + std::cout << "Multi-threaded pipelined Chia k" + std::to_string(KMAX) + " plotter"; #ifdef GIT_COMMIT_HASH std::cout << " - " << GIT_COMMIT_HASH; #endif @@ -485,18 +488,21 @@ int _main(int argc, char** argv) std::cout << std::endl << "Process has been interrupted, waiting for copy/rename operations to finish ..." << std::endl; break; } - std::cout << "Crafting plot " << i+1 << " out of " << num_plots << std::endl; + std::cout << "Crafting plot " << i+1 << " out of " << num_plots + << " (" << get_date_string_ex("%Y/%m/%d %H:%M:%S") << ")" << std::endl; const auto out = create_plot( k, port, num_threads, log_num_buckets, log_num_buckets_3, - pool_key, puzzle_hash, farmer_key, tmp_dir, tmp_dir2); + pool_key, puzzle_hash, farmer_key, tmp_dir, tmp_dir2, directout ? final_dir : tmp_dir); if(final_dir != tmp_dir) { - const auto dst_path = final_dir + out.params.plot_name + ".plot"; - std::cout << "Started copy to " << dst_path << std::endl; - copy_thread.take_copy(std::make_pair(out.plot_file_name, dst_path)); - if(waitforcopy) { - copy_thread.wait(); + if(!directout) { + const auto dst_path = final_dir + out.params.plot_name + ".plot"; + std::cout << "Started copy to " << dst_path << std::endl; + copy_thread.take_copy(std::make_pair(out.plot_file_name, dst_path)); + if(waitforcopy) { + copy_thread.wait(); + } } } else if(tmptoggle) {