WIP: issue #213 - load chunks from a dat, prototype

lib/create-dat-node.js

@@ -0,0 +1,65 @@
+var fs = require('fs')
+var url = require('url')
+var path = require('path')
+var Dat = require('dat-node')
+var smalltalk = require('smalltalk')
+var mirror = require('mirror-folder')
+var ram = require('random-access-memory')
+
+module.exports = function (collection, index, context, fileObject, cb) {
+  smalltalk.prompt('Enter Dat url', '', 'dat://').then(function (value) {
+    var datDetails = url.parse(value)
+    var datHash = datDetails.hostname
+
+    var folderPath = fileObject.resolvePath(datHash)
+    fetchDat(datHash, folderPath, function (err) {
+      var path = fileObject.resolvePath(datHash + '/index.json')
+      var externalDescriptor  = {
+        node: 'Node 1',
+        scale: '$global',
+        node: 'externalChunk',
+        src: fileObject.relative(path),
+        id: 'dat ' + datHash.substring(0, 5),
+        routes: {output: '$default'}
+      }
+      var node = collection.insert(externalDescriptor, index)
+      awaitResolve(node.loaded, function () {
+        cb && cb(null, node)
+      })
+    })
+  })
+}
+
+function fetchDat(hash, folderPath, cb) {
+  fs.mkdirSync(folderPath)
+  Dat(ram, {key: hash, sparse: true}, function (err, dat) {
+    if (err) return cb(err)
+    var network = dat.joinNetwork()
+    network.once('connection', function () { console.log('Connected') })
+    dat.archive.metadata.update(function () {
+      var progress = mirror({fs: dat.archive, name: '/'}, folderPath, function (err) {
+        if (err) return cb(err)
+        dat.leaveNetwork()
+        dat.close(cb)
+      })
+      progress.on('put', function (src) {
+        console.log('Downloading', src.name)
+      })
+    })
+    console.log(`Downloading: ${dat.key.toString('hex')}\n`)
+  })
+}
+
+
+function awaitResolve (obs, cb) {
+  if (obs()) {
+    cb(obs())
+  } else {
+    var release = obs(function (value) {
+      if (release) {
+        release()
+        cb(value)
+      }
+    })
+  }
+}

lib/spawn-node.js

@@ -1,6 +1,7 @@
 var resolveAvailable = require('lib/resolve-available')
 var extend = require('xtend')
 var getBaseName = require('path').basename
+var createDatNode = require('./create-dat-node')
 
 module.exports = function (collection, nodeName, index, cb) {
   if (typeof index === 'function') {
@@ -17,6 +18,7 @@ module.exports = function (collection, nodeName, index, cb) {
   var lookup = context.nodeInfo.lookup
   var info = lookup[nodeName]
   if (info) {
+    if (info.name === 'dat') return createDatNode(collection, index, context, fileObject, cb)
     var name = (info.name || 'Node') + ' 1'
     var spawnDescriptor = (typeof info.spawn === 'function')
       ? info.spawn(context)

nodes/dat-chunk/index.js

@@ -0,0 +1,7 @@
+var randomColor = require('lib/random-color')
+module.exports = {
+  name: 'dat',
+  node: 'dat-chunk',
+  group: 'simpleChunks',
+  description: 'Load a chunk from dat p2p'
+}

package.json

@@ -27,6 +27,7 @@
     "bopper": "~2.11.0",
     "brace": "~0.8.0",
     "bulk-require": "~0.2.1",
+    "dat-node": "^3.5.3",
     "decibels": "^1.0.0",
     "deep-equal": "^0.2.1",
     "ejs": "^2.5.6",
@@ -40,6 +41,7 @@
     "json-query": "^1.4.0",
     "make-distortion-curve": "^1.0.0",
     "micro-css": "~2.0.0",
+    "mirror-folder": "^2.1.1",
     "mkdirp": "^0.5.0",
     "mutant": "^3.20.0",
     "ncp": "^2.0.0",
@@ -53,8 +55,10 @@
     "pull-cat": "^1.1.11",
     "pull-stream": "^3.4.5",
     "pull-stream-to-stream": "github:mmckegg/pull-stream-to-stream#e436acee18b71af8e71d1b5d32eee642351517c7",
+    "random-access-memory": "^2.4.0",
     "readable-blob-stream": "^1.1.0",
     "scroll-into-view": "~1.3.1",
+    "smalltalk": "^2.4.1",
     "strftime": "^0.9.2",
     "tap-tempo": "~0.0.0",
     "teoria": "~0.4.0",