Adding b64 and protobuf support for DHT

doc/luaipfs.md

@@ -170,11 +170,22 @@ ___
 
 ###### Args:
 > key (string) : key (multihash) to search for
-> mode (string) : mode can be "lua", "raw" (protobuf) or "b64" (base64)
+> mode (optional, string) : mode can be "lua", "raw" (protobuf) or "b64" (base64). Default to "raw".  
 
 ###### Return:
 > Arbitrary data value (IPNS record for /ipns in lua, raw or b64 encoding).
 
+```lua
+--For "lua" mode :
+ret = {
+   signature = "bin-string",
+   validityType = "string",
+   sequence = "number",
+   validity = "string",
+   value = "string"
+}
+```
+
 > Or false and an error.
 
 
@@ -188,7 +199,7 @@ ___
 
 ###### Args:
 > key (string) : peer id (or another user's public key hash) 
-> value (string) : IPNS record (protobuf .pb file)
+> value (string) : IPNS record (protobuf .pb filepath)
 
 ###### Return:
 > A list of peers/neighbors (node id) where we have store value in the dht. 

examples/dht_advanced.lua

@@ -5,36 +5,72 @@ local ipfs = lipfs:new()
 
 
 
---(Re)provide/announce to the DHT that we have a file corresponding to the given key (file should exist in ipfs db). 
---Key correspond to the ipfs documentation directory (pined by default with go-ipfs).
+--Retrieve raw value of IPNS record (protobuf binary format) from the dht and write it to a file 
 
---[[
-local key = "QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc"
+local key = "/ipns/k51qzi5uqu5dl8ovmsw032wd9manqo8asfwdw3m6e84qyng2qbspv97dap5bpd"
+local f = io.open("ipns_record.pb", "w+b")
+if not f then return end
 
-local ret, err = ipfs:dht_provide(key)
+print("Trying to write ipns_record.pb ...")
+local ret, err = ipfs:dht_get(key)
 if not ret then
    print(err)
    return
 end
 
-print(ret)
+f:write(ret)
+f:close()
+print("Sucess !")
+
+
+
+
+--[[
+   Retrieve and convert IPNS record (protobuf) to Lua table, print some informations
+   from IPNS record and try to list files from ipfs link. 
 ]]
 
+local key = "/ipns/k51qzi5uqu5dl8ovmsw032wd9manqo8asfwdw3m6e84qyng2qbspv97dap5bpd"
+local ret, err = ipfs:dht_get(key, "lua")
+if not ret then
+   print(err)
+   return
+end
+
+print("IPNS record is valid until : " .. ret.validity)
+print("IPFS link attached to it : " .. ret.value)
+
+local files = ipfs:ls(ret.value)
+for i, file in pairs(files) do
+   print("---File #" .. i)
+   for k, v in pairs(file) do
+      print(k, v)
+   end
+end
+
 
---Retrieve raw value (IPNS record / protobuf) from the dht and write it to a file 
+
+
+--[[
+   Put key/value on the dht using ipns (user's id/public key) as key and previously saved ipns_record.pb as value.
+   It will update/replace old IPNS record and return the list of peers who received the new record (I think). 
+]]
 
 local key = "/ipns/k51qzi5uqu5dl8ovmsw032wd9manqo8asfwdw3m6e84qyng2qbspv97dap5bpd"
-local f = io.open("protobuf.pb", "wb")
-if not f then return end
 
-local ret, err = ipfs:dht_get(key)
-if not ret then
+print("Trying to put " .. key .. " and ipns_record.pb in the dht ...")
+
+local ret, err = ipfs:dht_put(key, "ipns_record.pb")
+if not ret then 
    print(err)
    return
 end
 
-f:write(ret)
-f:close()
+print("List of peers who received new IPNS record :")
+for k, v in pairs(ret) do
+   print(k, v)
+end
+
 
 
 return 0

examples/dht_example.lua

@@ -47,7 +47,6 @@ end
 --(Re)provide/announce to the DHT that we have a file corresponding to the given key (file should exist in ipfs db). 
 --Key correspond to the ipfs documentation directory (pined by default with go-ipfs).
 
---[[
 local key = "QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc"
 
 local ret, err = ipfs:dht_provide(key)
@@ -57,7 +56,7 @@ if not ret then
 end
 
 print(ret)
-]]
+
 
 
 

examples/multithreading_example.lua

@@ -23,7 +23,7 @@ f2 = lanes.gen("*", function ()
    local lipfs = require("luaipfs")
 
    local ipfs = lipfs:new()
-   local peer_id = "QmdcDLV5czbEBB1RoYvMxsYUt8KRp5g8GWKsg86cHb2ByD"
+   local peer_id = "QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"
 
    local peer, err = ipfs:dht_findpeer(peer_id)
    if not peer then
@@ -42,8 +42,8 @@ f3 = lanes.gen("*", function ()
 
    local ipfs = lipfs:new()
 
-   local key = "/ipns/QmQFwJEPLj3PVMEZKpk2oM96Teg3sKxMZSgA1RwYHmpgx2"
-   local data, err = ipfs:dht_get(key)
+   local key = "/ipns/k51qzi5uqu5dl8ovmsw032wd9manqo8asfwdw3m6e84qyng2qbspv97dap5bpd"
+   local data, err = ipfs:dht_get(key, "b64")
    if not data then 
       print(err)
       return false

examples/retrieve_files.lua

@@ -9,9 +9,9 @@ local ipfs = lipfs:new()
 
 --Retrieve a file in data buffer
 
-local file = io.open("dot.gif", "w+")
+local file = io.open("quick-start", "w+")
 local data, err = 
-   ipfs:cat("/ipfs/QmbZN6yqZF51dZVDP6NR9pksrwh3Tdh1wEwWZ75vRHd1m4/subdir/dotlink.gif")
+   ipfs:cat("/ipfs/QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc/quick-start")
 if err then print(err) elseif file then file:write(data) end
 
 file:close()
@@ -22,10 +22,10 @@ file:close()
 
 
 
---Retrieve a file to path "qemu.png"
+--Retrieve a file to path "help"
 
 local ret, err = 
-   ipfs:cat("/ipfs/QmbZN6yqZF51dZVDP6NR9pksrwh3Tdh1wEwWZ75vRHd1m4/qemu_glitch.png", "qemu.png")
+   ipfs:cat("/ipfs/QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc/help", "help")
 if err then print(err) end
 
 
@@ -37,7 +37,7 @@ if err then print(err) end
 --Retrieve an archive (file.s or directories) to path
 
 local ret, err =
-   ipfs:get("/ipfs/QmbZN6yqZF51dZVDP6NR9pksrwh3Tdh1wEwWZ75vRHd1m4/subdir", "myarchive.tar")
+   ipfs:get("/ipfs/QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc", "myarchive.tar")
 if err then print(err) end
 
 
@@ -49,7 +49,7 @@ if err then print(err) end
 --Set a callback function cb_func that will be call many times (until there is no more data) 
 --during adv_cat function call.
 
-local file = io.open("dot_copy.gif", "w+")
+local file = io.open("quick-start-copy", "w+")
 local curr_size = 0
 local function cb_func (ipfs_path, data, filesize)
    curr_size = curr_size + #data
@@ -61,7 +61,7 @@ end
 
 
 local ret, err = 
-   ipfs:adv_cat("/ipfs/QmbZN6yqZF51dZVDP6NR9pksrwh3Tdh1wEwWZ75vRHd1m4/subdir/dotlink.gif", cb_func)
+   ipfs:adv_cat("/ipfs/QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc/quick-start", cb_func)
 if not ret then print(err) end
 
 file:close()

luaipfs/dht.lua

@@ -111,7 +111,7 @@ end
 
 
 
-function dht:dht_get (key)
+function dht:dht_get (key, mode)
    if not key or type(key) ~= "string" then
       return false, "Invalid parameter #1 (string expected, got " .. type(key) .. ")"
    end
@@ -140,30 +140,58 @@ function dht:dht_get (key)
    end
 
    if not value then return false, "no value found in dht" end
+
+   --Select return value 
+   mode = mode or "raw"
+   if mode == "raw" then
+      value = self.b64_decode(value)
+      if not value then return false, "invalid b64 encoding" end
+   elseif mode == "b64" then
+      value = value
+   elseif mode == "lua" then
+      value = self.pb_decode("IpnsEntry", self.b64_decode(value))
+      if not value then return false, "invalid protobuf schema" end
+   end
+
    return value
 end
 
 
 
 --Note, https://github.com/ipfs/go-ipfs/blob/master/namesys/pb/namesys.proto
 
-function dht:dht_put (key, value)
+function dht:dht_put (key, filepath)
    if not key or type(key) ~= "string" then
       return false, "Invalid parameter #1 (string expected, got " .. type(key) .. ")"
-   elseif not value or type(value) ~= "string" then
-      return false, "Invalid parameter #2 (string expected, got " .. type(value) .. ")"
+   elseif not filepath or type(filepath) ~= "string" then
+      return false, "Invalid parameter #2 (string expected, got " .. type(filepath) .. ")"
    end
 
-   local res, http_ret, err = self.http:post("/api/v0/dht/put?arg=" .. key .. "&arg=" .. value)
-   if not res then
-      if err and not is_json(err) then return false, err end
-      return false, err and (json.decode(err)).Message or http_ret
+   local fattr, err = lfs.attributes(filepath)
+   if not fattr then return false, err end
+   if fattr.mode ~= "file" then
+      return false, "Invalid parameter (file expected, got " .. fattr.mode .. ")"
    end
 
-   print(res)
 
-   --Parse each line from answer
+   --Prepare files list before calling post_multipart_data
 
+   local filename = string.gsub(filepath, ".+/", "")
+   local tfiles = {}
+   tfiles[1] = {}
+   tfiles[1].filename = filename
+   tfiles[1].path = filepath
+   tfiles[1].mode = fattr.mode
+
+
+   local api_call = "/api/v0/dht/put?arg=" .. key
+   local res, http_ret = self.http:post_multipart_data(api_call, tfiles)
+   if not res then
+      return false, http_ret
+   end
+
+   --Parse each line from answer
+
    local t = {}
    for line in string.gmatch(res, "[^\n]+") do
       t[#t + 1] = json.decode(line)

luaipfs/init.lua

@@ -11,11 +11,15 @@
 local http = require("luaipfs.http")
 local json = require("json")
 local lfs = require("lfs")
+local b64 = require("base64")
+local pb = require("pb")
+local protoc = require("protoc")
 local base = require("luaipfs.base")
 local bitswap = require("luaipfs.bitswap")
 local dht = require("luaipfs.dht")
 local pin = require("luaipfs.pin")
 local swarm = require("luaipfs.swarm")
+local pb_schema = require("luaipfs.protobuf")
 
 
 
@@ -37,6 +41,10 @@ local ipfs = {
    dht_query = dht.dht_query,
    dht_provide = dht.dht_provide,
    bitswap_ledger = bitswap.bitswap_ledger,
+   b64_decode = b64.decode,
+   b64_encode = b64.encode,
+   pb_decode = pb.decode,
+   pb_encode = pb.encode
 }
 
 
@@ -61,6 +69,13 @@ function ipfs:new (o)
    o.http = http:new({server = o.server, port = o.port, timeout = o.timeout})
 
 
+   --Attach new protoc to it and compile/load protobuf schemas
+
+   o.protoc = protoc.new()
+   assert(o.protoc:load(pb_schema.ipns.data))
+
+
+
    return o
 end
 

luaipfs/protobuf.lua

@@ -0,0 +1,31 @@
+
+
+-- See https://github.com/ipfs/go-ipfs/blob/1e9e2f453c435272433a96ebfba5c4319cc08534/namesys/pb/namesys.proto
+
+local ipns_entry = [[
+   message IpnsEntry {
+      enum ValidityType {
+         EOL = 0;
+      }
+      required bytes value = 1;
+      required bytes signature = 2;
+
+      optional ValidityType validityType = 3;
+      optional bytes validity = 4;
+
+      optional uint64 sequence = 5;
+
+      optional uint64 ttl = 6;
+   }
+]]
+
+
+local pb_schema = {
+   ["ipns"] = {
+      name = "IpnsEntry",
+      data = ipns_entry
+   }
+}
+
+
+return pb_schema