Update testnet genesis

.gitignore

@@ -1,10 +1,11 @@
-*/__pycache__/
+**/__pycache__/
 .cache/
-tests/*/__pycache__/
+.pytest_cache/
 */*~
 *.#*
 *#
 *~
+docs/_build
 /build
 /dist
 /electrumx.egg-info

.travis.yml

@@ -4,18 +4,41 @@ language: python
 before_install:
   - sudo add-apt-repository -y ppa:giskou/librocksdb
   - sudo apt-get -qq update
-  - sudo apt-get install -yq libleveldb-dev librocksdb libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev
+  - sudo apt-get install -yq libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev
+  - sudo apt-get install -yq --allow-unauthenticated librocksdb
+  - wget https://launchpad.net/ubuntu/+archive/primary/+files/leveldb_1.20.orig.tar.gz
+  - tar -xzvf leveldb_1.20.orig.tar.gz
+  - pushd leveldb-1.20 && make && sudo mv out-shared/libleveldb.* /usr/local/lib && sudo cp -R include/leveldb /usr/local/include && sudo ldconfig && popd
 python:
   - "3.6"
   - "nightly"
 # command to install dependencies
 install:
   - pip install aiohttp
+  - pip install aiorpcX
   - pip install ecdsa
   - pip install plyvel
+  - pip install pycodestyle
+  - pip install pylru
   - pip install pyrocksdb
   - pip install pytest-cov
-  - pip install python-coveralls
+  - pip install Sphinx
+# hashes
+  - pip install tribus-hash
+  - pip install blake256
+  - pip install x11_hash
+  - pip install xevan_hash
+  - pip install quark_hash
+  - pip install git+https://github.com/goacoincore/neoscrypt
+  - pip install git+https://github.com/motioncrypto/x16r_hash
 # command to run tests
-script: pytest --cov=server --cov=lib --cov=wallet
-after_success: coveralls
+script:
+  - pytest --cov=electrumx
+  - pycodestyle electrumx/server/*.py electrumx/lib/*.py
+  - sh -c "cd docs && make html"
+# Dont report coverage from nightly
+after_success:
+  - if [[ $(python3 -V 2>&1) == *"Python 3.6"* ]]; then
+      pip install python-coveralls;
+      coveralls;
+    fi

README.rst

@@ -13,232 +13,12 @@ For a future network with bigger blocks.
   :Language: Python (>= 3.6)
   :Author: Neil Booth
 
-Getting Started
-===============
+Documentation
+=============
 
 See `docs/HOWTO.rst`_.
 There is also an `installer`_ available that simplifies the installation on various Linux-based distributions.
 
 .. _installer: https://github.com/Joe0/electrumx-installer
 
 Features
-========
-
-- Efficient, lightweight reimplementation of electrum-server
-- Fast synchronization of bitcoin mainnet from Genesis.  Recent
-  hardware should synchronize in well under 24 hours.  The fastest
-  time to height 448k (mid January 2017) reported is under 4h 30m.  On
-  the same hardware JElectrum would take around 4 days and
-  electrum-server probably around 1 month.
-- The full Electrum protocol is implemented.  The only exception is
-  the blockchain.address.get_proof RPC call, which is not used by
-  Electrum GUI clients, and can only be invoked from the command line.
-- Various configurable means of controlling resource consumption and
-  handling denial of service attacks.  These include maximum
-  connection counts, subscription limits per-connection and across all
-  connections, maximum response size, per-session bandwidth limits,
-  and session timeouts.
-- Minimal resource usage once caught up and serving clients; tracking the
-  transaction mempool appears to be the most expensive part.
-- Fully asynchronous processing of new blocks, mempool updates, and
-  client requests.  Busy clients should not noticeably impede other
-  clients' requests and notifications, nor the processing of incoming
-  blocks and mempool updates.
-- Daemon failover.  More than one daemon can be specified, and
-  ElectrumX will failover round-robin style if the current one fails
-  for any reason.
-- Peer discovery protocol removes need for IRC
-- Coin abstraction makes compatible altcoin and testnet support easy.
-
-Implementation
-==============
-
-ElectrumX does not do any pruning or throwing away of history.  I want
-to retain this property for as long as it is feasible, and it appears
-efficiently achievable for the forseeable future with plain Python.
-
-The following all play a part in making ElectrumX very efficient as a
-Python blockchain indexer:
-
-- aggressive caching and batching of DB writes
-- more compact and efficient representation of UTXOs, address index,
-  and history.  Electrum Server stores full transaction hash and
-  height for each UTXO, and does the same in its pruned history.  In
-  contrast ElectrumX just stores the transaction number in the linear
-  history of transactions.  For at least another 5 years this
-  transaction number will fit in a 4-byte integer, and when necessary
-  expanding to 5 or 6 bytes is trivial.  ElectrumX can determine block
-  height from a simple binary search of tx counts stored on disk.
-  ElectrumX stores historical transaction hashes in a linear array on
-  disk.
-- placing static append-only metadata indexable by position on disk
-  rather than in levelDB.  It would be nice to do this for histories
-  but I cannot think of a way.
-- avoiding unnecessary or redundant computations, such as converting
-  address hashes to human-readable ASCII strings with expensive bignum
-  arithmetic, and then back again.
-- better choice of Python data structures giving lower memory usage as
-  well as faster traversal
-- leveraging asyncio for asynchronous prefetch of blocks to mostly
-  eliminate CPU idling.  As a Python program ElectrumX is unavoidably
-  single-threaded in its essence; we must keep that CPU core busy.
-
-Python's ``asyncio`` means ElectrumX has no (direct) use for threads
-and associated complications.
-
-
-Roadmap
-=======
-
-- offloading more work of wallet synchronization to the client
-- supporting better client privacy
-- wallet server engine
-- new features such as possibly adding label server functionality
-- potentially move some functionality to C or C++
-
-
-ChangeLog
-=========
-
-IMPORTANT: version 1.2 changes script hash indexing in the database,
-so you will need to rebuild your databases from scratch.  Running this
-version will refuse to open the DB and not corrupt it, so you can
-revert to 1.1.x if you wish.  The initial synchronisation process
-should be around 10-15% faster than 1.1, owing to this change and
-Justin Arthur's optimisations from 1.1.1.
-
-Version 1.2.1
--------------
-
-- remove IRC support.  Most coins had empty IRC channels.  Those that
-  don't have peers populated.
-- use estimatesmartfee RPC call if available (SomberNight)
-- new/updated coins: Emercoin (Sergii Vakula), Bitcoin Gold (erasmospunk),
-  Monacoin testnet (Wakiyama P), sibcoin (53r63rn4r), Komodo and Monaize
-  (cipig), Hush (Duke Leto)
-- doc updates (fr3aker)
-- issues fixed: `#302`_
-
-Version 1.2
------------
-
-- separate P2PKH from P2PK entries in the history and UTXO databases.
-  These were previously amalgamated by address as that is what
-  electrum-server used to do.  However Electrum didn't handle P2PK
-  spends correctly and now the protocol admits subscriptions by script
-  hash there is no need to have these merged any more.
-
-For Bitcoin (BitcoinSegwit/mainnet) you can download a leveldb database
-synced up to block 490153 using this bittorrent magnet link (~24GB):
-    magnet:?xt=urn:btih:caa804f48a319b061be3884ac011656c27121a6f&dn=electrumx_1.2_btc_leveldb_490153
-
-Version 1.1.2
--------------
-
-- PEER_DISCOVERY environment variable is now tri-state (fixes
-  `#287`_).  Please check your setting as its meaning has changed
-  slightly.
-- fix listunspent protocol methods to remove in-mempool spends (fixes
-  `#277`_).
-- improved environment variable handling
-- EMC2 update (cipig), Monacoin update (cryptocoin-junkey),
-  Canada Ecoin (koad)
-- typo fixes, Bitcoin testnet peers updates (SomberNight)
-
-Version 1.1.1
--------------
-
-- various refactorings, improvement of env var handling
-- update docs to match
-- various optimizations mainly affecting initial sync (Justin Arthur)
-- Dash fixes (cipig)
-- Add ALLOW_ROOT option (Luke Childs)
-- Add BitZeny support, update Monacoin (cryptocoin-junkey)
-
-Version 1.1
------------
-
-See the changelogs below for recent changes.  The most important is
-that for mainnet bitcoin **NET** must now be *mainnet* and you must
-choose a **COIN** from *BitcoinCash* and *BitcoinSegwit*.  Similarly
-for testnets.  These coins will likely diverge further in future so
-it's best they become separate coins now.
-
-- no longer persist peers, rediscover on restart
-- onion peers only reported if can connect; hard-coded exception removed
-- small fix for blockchain.transaction.broadcast
-
-Version 1.1pre2
----------------
-
-- peerdisc: handle protocol 1.1 server replies
-- issue `#251`_: fix protocol version reported in server.peers.subscribe
-- fix handling of failed transaction broadcast
-- fix typos (SomberNight)
-- doc and test updates
-- dash: return errors in JSON error field for protocol 1.1
-
-Version 1.1pre1
----------------
-
-Many changes, mostly to prepare for support of Electrum protocol 1.1
-which the next Electrum client release will use.
-
-*NOTE*: the **COIN** environment variable is now mandatory, and if you
-were running for any bitcoin flavour (Cash, Segwit, mainnet or
-testnet) you will need to update your **COIN** and **NET** environment
-variable settings as the old ones will no longer work.
-
-- implement protocol version 1.1 and update protocol documentation
-- rework lib/coins.py for the various bitcoin flavours
-- show protocol version in "sessions" ElectrumX RPC call
-- permit **HOST** envvar to be a comma-separated list
-- daemon abstraction refactoring (erasmospunk)
-- permit alternative event loop policies (based on suggestion / work
-  of JustinTArthur)
-- misc configuration updates (SubPar)
-- add Neblio support (neblioteam) and Bitbay (anoxxxy)
-- HOWTO.rst update for running on privileged port (EagleTM)
-- issues closed: exclude test dirs from installation (`#223`_).
-
-Version 1.0.17
---------------
-
-- fix #227 introduced in 1.0.16
-
-Version 1.0.16
---------------
-
-- updated server lists for post-fork.  If you are on the Segwit chain
-  you should have NET be "bitcoin-segwit", and if on the Bitcoin Cash chain
-  continue to use "mainnet".
-- binding address fix for multi-homed machines (mmouse)
-- listen to IPv4 and IPv6 local interfaces
-- add Fujicoin (fujicoin), Crown (Extreemist), RegTest (RCasatta),
-  Monacoin (cryptocoin-junkey)
-- bug fixes and updates (Kefkius, mmouse, thesamesam, cryptocoin-junkey,
-  jtarthur)
-
-Version 1.0.15
---------------
-
-- split server networks faster if a fork is detected
-- minor speedup
-- add Vertcoin support (erasmospunk)
-- update Faircoin (thokon00)
-
-
-**Neil Booth**  [email protected]  https://github.com/kyuupichan
-
-1BWwXJH3q6PRsizBkSGm2Uw4Sz1urZ5sCj
-
-LKaFk4KkVpw9pYoUpbckQSKKgCVC4oj78b
-
-.. _#223: https://github.com/kyuupichan/electrumx/issues/223
-.. _#251: https://github.com/kyuupichan/electrumx/issues/251
-.. _#277: https://github.com/kyuupichan/electrumx/issues/277
-.. _#287: https://github.com/kyuupichan/electrumx/issues/287
-.. _#302: https://github.com/kyuupichan/electrumx/issues/287
-.. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst
-.. _docs/ENVIRONMENT.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/ENVIRONMENT.rst
-.. _docs/PROTOCOL.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/PROTOCOL.rst

compact_history.py

@@ -31,40 +31,46 @@
 subsequent compactions will restart from the beginning.
 '''
 
+import asyncio
 import logging
 import sys
 import traceback
 from os import environ
 
-from server.env import Env
-from server.db import DB
+from electrumx import Env
+from electrumx.server.db import DB
 
 
-def compact_history():
+async def compact_history():
     if sys.version_info < (3, 6):
         raise RuntimeError('Python >= 3.6 is required to run ElectrumX')
 
     environ['DAEMON_URL'] = ''   # Avoid Env erroring out
     env = Env()
     db = DB(env)
+    await db.open_for_sync()
 
     assert not db.first_sync
+    history = db.history
     # Continue where we left off, if interrupted
-    if db.comp_cursor == -1:
-        db.comp_cursor = 0
+    if history.comp_cursor == -1:
+        history.comp_cursor = 0
 
-    db.comp_flush_count = max(db.comp_flush_count, 1)
+    history.comp_flush_count = max(history.comp_flush_count, 1)
     limit = 8 * 1000 * 1000
 
-    while db.comp_cursor != -1:
-        db._compact_history(limit)
+    while history.comp_cursor != -1:
+        history._compact_history(limit)
 
+    # When completed also update the UTXO flush count
+    db.set_flush_count(history.flush_count)
 
 def main():
     logging.basicConfig(level=logging.INFO)
     logging.info('Starting history compaction...')
+    loop = asyncio.get_event_loop()
     try:
-        compact_history()
+        loop.run_until_complete(compact_history())
     except Exception:
         traceback.print_exc()
         logging.critical('History compaction terminated abnormally')

contrib/daemontools/env/ELECTRUMX

@@ -1 +1 @@
-/path/to/electrumx_server.py
+/path/to/electrumx_server

contrib/daemontools/log/run

@@ -1,2 +1,2 @@
 #!/bin/sh
-exec multilog t s500000 n10 /path/to/log/dir
+exec multilog s500000 n10 /path/to/log/dir

contrib/daemontools/run

@@ -1,3 +1,5 @@
 #!/bin/sh
 echo "Launching ElectrumX server..."
-exec 2>&1 envdir ./env /bin/sh -c 'setuidgid $USERNAME python3 $ELECTRUMX'
+USERNAME=$(envdir ./env printenv USERNAME)
+ELECTRUMX=$(envdir ./env printenv ELECTRUMX)
+ulimit -n 4000 && exec 2>&1 envdir ./env envuidgid $USERNAME python3 $ELECTRUMX

contrib/python3.6/python-3.6.sh

@@ -6,7 +6,7 @@
 sudo add-apt-repository ppa:jonathonf/python-3.6
 sudo apt-get update && sudo apt-get install python3.6 python3.6-dev
 
-cd /home/username
+cd ~
 git clone https://github.com/kyuupichan/electrumx.git
 cd electrumx
 sudo python3.6 setup.py install

contrib/raspberrypi3/run_electrumx.sh

@@ -4,7 +4,7 @@
 ###############
 
 # configure electrumx
-export COIN=Bitcoin
+export COIN=BitcoinSegwit
 export DAEMON_URL=http://rpcuser:[email protected]
 export NET=mainnet
 export CACHE_MB=400
@@ -25,7 +25,7 @@ export RPC_PORT=8000
 
 # run electrumx
 ulimit -n 10000
-/usr/local/bin/electrumx_server.py 2>> /home/username/.electrumx/electrumx.log >> /home/username/.electrumx/electrumx.log &
+/usr/local/bin/electrumx_server 2>> /home/username/.electrumx/electrumx.log >> /home/username/.electrumx/electrumx.log &
 
 ######################
 # auto-start electrumx

contrib/systemd/electrumx.conf

@@ -5,5 +5,5 @@ DB_DIRECTORY = /db
 # Bitcoin Node RPC Credentials
 DAEMON_URL = http://username:password@hostname:port/
 
-# See https://github.com/kyuupichan/electrumx/blob/master/docs/ENVIRONMENT.rst for
+# See http://electrumx.readthedocs.io/en/latest/environment.html for
 # information about other configuration settings you probably want to consider.