Merge branch 'develop'

.gitlab-ci.yml

@@ -8,8 +8,8 @@ build-amd64-debian-jessie:
   image: debian:jessie
   script:
     - ./gitlab-build.sh
-    - mkdir -p built-packages/debian_jessie/
-    - mv ../*.deb built-packages/debian_jessie/
+    - mkdir -p built-packages/jessie/
+    - mv ../*.deb built-packages/jessie/
   artifacts:
     paths:
       - built-packages/*
@@ -22,8 +22,8 @@ build-amd64-debian-stretch:
   image: debian:stretch
   script:
     - ./gitlab-build.sh
-    - mkdir -p built-packages/debian_stretch/
-    - mv ../*.deb built-packages/debian_stretch/
+    - mkdir -p built-packages/stretch/
+    - mv ../*.deb built-packages/stretch/
   artifacts:
     paths:
       - built-packages/*
@@ -36,8 +36,8 @@ build-amd64-ubuntu-xenial:
   image: ubuntu:xenial
   script:
     - ./gitlab-build.sh
-    - mkdir -p built-packages/ubuntu_xenial/
-    - mv ../*.deb built-packages/ubuntu_xenial/
+    - mkdir -p built-packages/xenial/
+    - mv ../*.deb built-packages/xenial/
   artifacts:
     paths:
       - built-packages/*
@@ -50,8 +50,8 @@ build-amd64-ubuntu-bionic:
   image: ubuntu:bionic
   script:
     - ./gitlab-build.sh
-    - mkdir -p built-packages/ubuntu_bionic/
-    - mv ../*.deb built-packages/ubuntu_bionic/
+    - mkdir -p built-packages/bionic/
+    - mv ../*.deb built-packages/bionic/
   artifacts:
     paths:
       - built-packages/*
@@ -64,8 +64,8 @@ build-amd64-ubuntu-cosmic:
   image: ubuntu:cosmic
   script:
     - ./gitlab-build.sh
-    - mkdir -p built-packages/ubuntu_cosmic/
-    - mv ../*.deb built-packages/ubuntu_cosmic/
+    - mkdir -p built-packages/cosmic/
+    - mv ../*.deb built-packages/cosmic/
   artifacts:
     paths:
       - built-packages/*
@@ -87,13 +87,13 @@ build-amd64-fedora-29:
   only:
     - tags
 
-build-amd64-fedora-28:
+build-amd64-fedora-30:
   stage: build
-  image: fedora:28
+  image: fedora:30
   script:
-    - ./gitlab-build-rpm.sh fedora28
-    - mkdir -p built-packages/fedora_28/
-    - mv ~/rpmbuild/RPMS/x86_64/*.rpm built-packages/fedora_28/
+    - ./gitlab-build-rpm.sh fedora30
+    - mkdir -p built-packages/fedora_30/
+    - mv ~/rpmbuild/RPMS/x86_64/*.rpm built-packages/fedora_30/
   artifacts:
     paths:
       - built-packages/*
@@ -133,39 +133,39 @@ run-unit-tests-ubuntu-xenial:
   stage: test
   image: ubuntu:xenial
   script:
-    - ./run-tests.sh ubuntu_xenial
+    - ./run-tests.sh xenial
   only:
     - tags
 
 run-unit-tests-ubuntu-bionic:
   stage: test
   image: ubuntu:bionic
   script:
-    - ./run-tests.sh ubuntu_bionic
+    - ./run-tests.sh bionic
   only:
     - tags
 
 run-unit-tests-ubuntu-cosmic:
   stage: test
   image: ubuntu:cosmic
   script:
-    - ./run-tests.sh ubuntu_cosmic
+    - ./run-tests.sh cosmic
   only:
     - tags
 
 run-unit-tests-debian-stretch:
   stage: test
   image: debian:stretch
   script:
-    - ./run-tests.sh debian_stretch
+    - ./run-tests.sh stretch
   only:
     - tags
 
 run-unit-tests-debian-jessie:
   stage: test
   image: debian:jessie
   script:
-    - ./run-tests.sh debian_jessie
+    - ./run-tests.sh jessie
   only:
     - tags
 
@@ -177,11 +177,11 @@ run-unit-tests-fedora-29:
   only:
     - tags
 
-run-unit-tests-fedora-28:
+run-unit-tests-fedora-30:
   stage: test
-  image: fedora:28
+  image: fedora:30
   script:
-    - ./run-tests-rpm.sh fedora_28
+    - ./run-tests-rpm.sh fedora_30
   only:
     - tags
 

AUTHORS

@@ -15,4 +15,7 @@ Thanks to Sergey Cherepanov for adding support for lz4.
 Thanks to Mingwei Zhang for adding ability to resume HTTP reads and fixing
 a ton of lazy error checking issues.
 
+Thanks to Derrick Lyndon Pallas for fixing an uninitialised memory error
+when closing a wandio writer.
+
 Thanks to Brad Cowie for packaging libwandio for Debian.

ChangeLog

@@ -1,3 +1,12 @@
+Version 4.2.0
+=============
+ * Include wandio version number in the HTTP user-agent
+ * Improved error detection and handling when reading HTTP
+ * Replaced 'off_t' parameters and return values with 'int64_t' in
+   recently added API functions.
+ * Fixed potential uninitialised memory error when closing a wandio writer.
+ * Export symbols for all format-specific open functions.
+
 Version 4.1.2
 =============
  * Fix buffer overflow bug in the swift reading code (thanks Alistair).

README

@@ -1,4 +1,4 @@
-WANDIO 4.1.2
+WANDIO 4.2.0
 
 ---------------------------------------------------------------------------
 Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand.

bintray-upload.sh

@@ -4,7 +4,7 @@ set -e -o pipefail
 
 BINTRAY_DEB_REPO="wand/general"
 BINTRAY_RPM_REPO="wand/general-rpm"
-BINTRAY_LICENSE="LGPL-3"
+BINTRAY_LICENSE="LGPL-3.0"
 
 apt-get update && apt-get install -y curl util-linux
 
@@ -39,16 +39,24 @@ for path in `find built-packages/ -maxdepth 1 -type d`; do
         if [ "$ext" = "rpm" ]; then
             rev_filename=`echo ${pkg_filename} | rev`
 
-            if [[ "$1" =~ centos_* ]]; then
-                pkg_name=`echo ${rev_filename} | cut -d '-' -f4- | rev`
-                pkg_version=`echo ${rev_filename} | cut -d '-' -f1-3 | rev | cut -d '.' -f1-3`
+            if [[ ${linux_version} =~ centos_* ]]; then
+                # centos
+                pkg_dist="centos"
+
             else
-                pkg_name=`echo ${rev_filename} | cut -d '-' -f3- | rev`
-                pkg_version=`echo ${rev_filename} | cut -d '-' -f1-2 | rev | cut -d '.' -f1-3`
+                # fedora
+                pkg_dist="fedora"
             fi
 
+            pkg_name=`echo ${rev_filename} | cut -d '-' -f3- | rev`
+            pkg_version=`echo ${rev_filename} | cut -d '-' -f1-2 | rev | cut -d '.' -f1-3`
+            pkg_arch=`echo ${rev_filename} | cut -d '.' -f2 | rev`
+            pkg_rel=`echo ${rev_filename} | cut -d '.' -f3 | rev`
+            releasever="${pkg_rel:2}"
+
+
             jfrog bt package-create --licenses ${BINTRAY_LICENSE} --vcs-url ${CI_PROJECT_URL} ${BINTRAY_RPM_REPO}/${pkg_name} || true
-            jfrog bt upload ${deb} ${BINTRAY_RPM_REPO}/${pkg_name}/${pkg_version}
+            jfrog bt upload ${deb} ${BINTRAY_RPM_REPO}/${pkg_name}/${pkg_version} ${pkg_dist}/${releasever}/${pkg_arch}/
 
         fi
     done

configure.ac

@@ -3,11 +3,11 @@
 # Now you only need to update the version number in two places - below,
 # and in the README
 
-AC_INIT([wandio],[4.1.2],[[email protected]],[wandio])
+AC_INIT([wandio],[4.2.0],[[email protected]],[wandio])
 
 WANDIO_MAJOR=4
-WANDIO_MID=1
-WANDIO_MINOR=2
+WANDIO_MID=2
+WANDIO_MINOR=0
 
 # OpenSolaris hides libraries like libncurses in /usr/gnu/lib, which is not
 # searched by default - add it to LDFLAGS so we at least have a chance of

debian/changelog

@@ -1,3 +1,14 @@
+libwandio1 (4.2.0-1) unstable; urgency=medium
+
+  * Include wandio version number in the HTTP user-agent
+  * Improved error detection and handling when reading HTTP
+  * Replaced 'off_t' parameters and return values with 'int64_t' in
+    recently added API functions.
+  * Fixed potential uninitialised memory error when closing a wandio writer.
+  * Export symbols for all format-specific 'open' functions.
+
+ -- Shane Alcock <[email protected]>  Fri, 10 May 2019 13:31:49 +1200
+
 libwandio1 (4.1.2-1) unstable; urgency=medium
 
   * Fix swift buffer overflow bug

gitlab-build-rpm.sh

@@ -3,6 +3,7 @@
 set -x -e -o pipefail
 
 export QA_RPATHS=$[ 0x0001 ]
+SOURCENAME=`echo ${CI_COMMIT_REF_NAME} | cut -d '-' -f 1`
 
 if [ "$1" = "centos7" ]; then
         yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
@@ -14,7 +15,7 @@ if [ "$1" = "centos6" ]; then
 fi
 
 if [[ "$1" =~ fedora* ]]; then
-        dnf install -y rpm-build rpmdevtools
+        dnf install -y rpm-build rpmdevtools which 'dnf-command(builddep)'
         dnf group install -y "C Development Tools and Libraries"
         dnf builddep -y rpm/libwandio1.spec
 else
@@ -26,7 +27,7 @@ fi
 rpmdev-setuptree
 
 ./bootstrap.sh && ./configure && make dist
-cp wandio-*.tar.gz ~/rpmbuild/SOURCES/${CI_COMMIT_REF_NAME}.tar.gz
+cp wandio-*.tar.gz ~/rpmbuild/SOURCES/${SOURCENAME}.tar.gz
 cp rpm/libwandio1.spec ~/rpmbuild/SPECS/
 
 cd ~/rpmbuild && rpmbuild -bb --define "debug_package %{nil}" SPECS/libwandio1.spec

gitlab-build.sh

@@ -6,6 +6,8 @@ export DEBEMAIL='[email protected]'
 export DEBFULLNAME='WAND Packaging'
 export DEBIAN_FRONTEND=noninteractive
 
+SOURCENAME=`echo ${CI_COMMIT_REF_NAME} | cut -d '-' -f 1`
+
 apt-get update
 apt-get install -y equivs devscripts dpkg-dev quilt curl apt-transport-https \
     apt-utils ssl-cert ca-certificates gnupg lsb-release debhelper git
@@ -19,6 +21,6 @@ chmod 644 /etc/apt/trusted.gpg.d/wand.gpg
 
 apt-get update
 
-dpkg-parsechangelog -S version | grep -q ${CI_COMMIT_REF_NAME} || debchange --newversion ${CI_COMMIT_REF_NAME} -b "New upstream release"
+dpkg-parsechangelog -S version | grep -q ${SOURCENAME} || debchange --newversion ${SOURCENAME} -b "New upstream release"
 mk-build-deps -i -r -t 'apt-get -f -y --force-yes'
 dpkg-buildpackage -b -us -uc -rfakeroot -j4

lib/Makefile.am

@@ -73,5 +73,5 @@ libwandio_la_SOURCES=wandio.c ior-peek.c ior-stdio.c ior-thread.c \
 
 AM_CPPFLAGS = @ADD_INCLS@
 libwandio_la_LIBADD = @LIBWANDIO_LIBS@
-libwandio_la_LDFLAGS=-version-info 5:2:1 @ADD_LDFLAGS@
+libwandio_la_LDFLAGS=-version-info 6:0:0 @ADD_LDFLAGS@
 

lib/ior-bzip.c

@@ -51,7 +51,7 @@ extern io_source_t bz_source;
 #define DATA(io) ((struct bz_t *)((io)->data))
 #define min(a, b) ((a) < (b) ? (a) : (b))
 
-io_t *bz_open(io_t *parent) {
+DLLEXPORT io_t *bz_open(io_t *parent) {
         io_t *io;
         if (!parent)
                 return NULL;

lib/ior-http.c

@@ -156,7 +156,9 @@ static int fill_buffer(io_t *io) {
                 struct timeval to;
                 // the following is adaped from docs/examples/fopen.c
                 to.tv_sec = 10, to.tv_usec = 0;  // 10 seconds
-                curl_multi_timeout(DATA(io)->multi, &curl_to);
+                if (curl_multi_timeout(DATA(io)->multi, &curl_to) != CURLM_OK) {
+                  return -1;
+                }
                 if (curl_to >= 0) {
                         to.tv_sec = curl_to / 1000;
                         if (to.tv_sec > 1)
@@ -168,8 +170,10 @@ static int fill_buffer(io_t *io) {
                 FD_ZERO(&fdw);
                 FD_ZERO(&fde);
 
-                /* FIXME: check return code */
-                curl_multi_fdset(DATA(io)->multi, &fdr, &fdw, &fde, &maxfd);
+                if (curl_multi_fdset(DATA(io)->multi, &fdr, &fdw, &fde,
+                                     &maxfd) != CURLM_OK) {
+                  return -1;
+                }
                 if (maxfd >= 0 &&
                     (rc = select(maxfd + 1, &fdr, &fdw, &fde, &to)) < 0)
                         break;
@@ -182,8 +186,10 @@ static int fill_buffer(io_t *io) {
                         nanosleep(&req, &rem);
                 }
                 curl_easy_pause(DATA(io)->curl, CURLPAUSE_CONT);
-                /* FIXME: check return code */
-                rc = curl_multi_perform(DATA(io)->multi, &n_running);
+                if (curl_multi_perform(DATA(io)->multi, &n_running) !=
+                    CURLM_OK) {
+                  return -1;
+                }
                 if (DATA(io)->total_length < 0) {
                         // update file length.
                         double cl;
@@ -195,6 +201,21 @@ static int fill_buffer(io_t *io) {
         } while (n_running &&
                  DATA(io)->l_buf < DATA(io)->m_buf - CURL_MAX_WRITE_SIZE);
 
+        // check if there were any errors from curl
+        struct CURLMsg *m = NULL;
+        do {
+          int msgq = 0;
+          m = curl_multi_info_read(DATA(io)->multi, &msgq);
+          if (m != NULL && m->data.result != CURLE_OK) {
+            // there was an error reading -- if this is the first
+            // read, then the wandio_create call will fail.
+            fprintf(stderr, "HTTP ERROR: %s (%d)\n",
+                    curl_easy_strerror(m->data.result),
+                    m->data.result);
+            return -1;
+          }
+        } while (m != NULL);
+
         if (DATA(io)->l_buf < DATA(io)->m_buf - CURL_MAX_WRITE_SIZE) {
                 if (DATA(io)->off0 + DATA(io)->p_buf >=
                     DATA(io)->total_length) {
@@ -219,7 +240,8 @@ static int fill_buffer(io_t *io) {
         return DATA(io)->l_buf;
 }
 
-io_t *http_open_hdrs(const char *filename, char **hdrs, int hdrs_cnt) {
+DLLEXPORT io_t *http_open_hdrs(const char *filename, char **hdrs, int hdrs_cnt)
+{
         io_t *io = malloc(sizeof(io_t));
         if (!io)
                 return NULL;
@@ -249,7 +271,7 @@ io_t *http_open_hdrs(const char *filename, char **hdrs, int hdrs_cnt) {
         return io;
 }
 
-io_t *http_open(const char *filename) {
+DLLEXPORT io_t *http_open(const char *filename) {
         return http_open_hdrs(filename, NULL, 0);
 }
 
@@ -272,9 +294,10 @@ io_t *init_io(io_t *io) {
         curl_easy_setopt(DATA(io)->curl, CURLOPT_VERBOSE, 0L);
         curl_easy_setopt(DATA(io)->curl, CURLOPT_NOSIGNAL, 1L);
         curl_easy_setopt(DATA(io)->curl, CURLOPT_WRITEFUNCTION, write_cb);
-        curl_easy_setopt(DATA(io)->curl, CURLOPT_SSL_VERIFYPEER, 0L);
-        curl_easy_setopt(DATA(io)->curl, CURLOPT_SSL_VERIFYHOST, 0L);
+        curl_easy_setopt(DATA(io)->curl, CURLOPT_SSL_VERIFYPEER, 1L);
+        curl_easy_setopt(DATA(io)->curl, CURLOPT_SSL_VERIFYHOST, 1L);
         curl_easy_setopt(DATA(io)->curl, CURLOPT_FOLLOWLOCATION, 1L);
+        curl_easy_setopt(DATA(io)->curl, CURLOPT_USERAGENT, "wandio/"PACKAGE_VERSION);
 
         /* for remote files, the buffer set to 2*CURL_MAX_WRITE_SIZE */
         DATA(io)->m_buf = CURL_MAX_WRITE_SIZE * 2;

lib/ior-lzma.c

@@ -51,7 +51,7 @@ extern io_source_t lzma_source;
 #define DATA(io) ((struct lzma_t *)((io)->data))
 #define min(a, b) ((a) < (b) ? (a) : (b))
 
-io_t *lzma_open(io_t *parent) {
+DLLEXPORT io_t *lzma_open(io_t *parent) {
         io_t *io;
         if (!parent)
                 return NULL;

lib/ior-peek.c

@@ -65,7 +65,7 @@ extern io_source_t peek_source;
 #define DATA(io) ((struct peek_t *)((io)->data))
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 
-io_t *peek_open(io_t *child) {
+DLLEXPORT io_t *peek_open(io_t *child) {
         io_t *io;
         if (!child)
                 return NULL;