When programming in JavaScript (or CoffeeScript) you sometimes face a situation where you need to complete multiple independant asynchronous methods before continuing. This situation crops up in web browsers but its much more common when writing server-side JavaScript, e.g. with Node.js - for example you might need to fetch from a database, fetch from a KVS, read a file and perform a remote HTTP request before outputting the compiled information to the end user.

One method of solving this issue is to chain the asynchronous calls, however this means that they’re run one after the other (serially) - and thus it will take longer to complete them. A better way would be to run them in parallel and have something track their completion. This is exactly what my very simple AsyncBatch class does:

1
2
3
4
5
6
7
8
9
10
11

class AsyncBatch extends EventEmitter
  constructor: ->
    @_complete = {}
    @_scheduled = {}

  wrap: (name,cb) ->
    @_scheduled[name] = true
    return =>
      @_complete[name] = cb.apply @, arguments
      if Object.keys(@_complete).length == Object.keys(@_scheduled).length
        @emit 'done', @_complete

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

var AsyncBatch;
var __hasProp = Object.prototype.hasOwnProperty, __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor; child.__super__ = parent.prototype; return child; };

AsyncBatch = (function() {

  __extends(AsyncBatch, EventEmitter);

  function AsyncBatch() {
    this._complete = {};
    this._scheduled = {};
  }

  AsyncBatch.prototype.wrap = function(name, cb) {
    var _this = this;
    this._scheduled[name] = true;
    return function() {
      _this._complete[name] = cb.apply(_this, arguments);
      if (Object.keys(_this._complete).length === Object.keys(_this._scheduled).length) {
        return _this.emit('done', _this._complete);
      }
    };
  };

  return AsyncBatch;

})();

To use AsyncBatch, just do the following

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

###
Create a new AsyncBatch instance
###
batch = new AsyncBatch

###
Wrap your callbacks with batch.wrap and a name
The name is used to store the result returned by your callback
(Don't forget to return the result you're interested in!)
###
delay 50, batch.wrap 'timer', ->
  return "Timer complete"

###
Add a completion handler that accepts `results` as a parameter.
  `results` is a JS object where the keys are the callback names 
  from above and the values are the return values of the
  callbacks.
###
batch.on 'done', (results) ->
  console.log "Batch complete, timer result: #{results.timer}"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

/*
Create a new AsyncBatch instance
*/

var batch;

batch = new AsyncBatch;

/*
Wrap your callbacks with batch.wrap and a name
The name is used to store the result returned by your callback
(Don't forget to return the result you're interested in!)
*/

delay(50, batch.wrap('timer', function() {
  return "Timer complete";
}));

/*
Add a completion handler that accepts `results` as a parameter.
  `results` is a JS object where the keys are the callback names 
  from above and the values are the return values of the
  callbacks.
*/

batch.on('done', function(results) {
  return console.log("Batch complete, timer result: " + results.timer);
});

NOTE: If your asynchronous method accepts both a success and failure callback then simply wrap both individually but ensure you use the same name for both.

NOTE: Other than the case in the previous NOTE, all callbacks should have different names.

A full example (including a stub EventEmitter implementation) follows:

I really like CoffeeScript. I really like JavaScript. I’m trying to get into the habit of writing more CoffeeScript since, I think, it allows me to express myself clearer and more concisely, leading to shorter code and (hopefully) less mistakes/bugs. It also makes jumping back into code a month later a much simpler affair since reading the code is so much easier.

The following JavaScript creates a new closure inside each iteration of a loop to store the value of the loop variable i into the closure’s local variable j so that we don’t just console.log(10) 10 times. Though this example is trivial it represents a common method for solving closure related issues in asynchronous code.

1
2
3
4
5
6
7
8

for (var i = 0; i < 10; i++) {
  (function(){
    var j = i;
    delay(0, function(){
      console.log(j);
    });
  })();
}

The following CoffeeScript does exactly the same thing, you can see the translation line by line from the JavaScript. However once you’ve learnt the syntax you can much more clearly see what is going on with the CoffeeScript than in the verbose and parenthesis-heavy JavaScript above.

1
2
3
4
5

for i in [0..10]
  do ->
    j = i
    delay 0, ->
      console.log j

1
2
3
4
5
6
7
8
9
10
11
12

var i, _fn;

_fn = function() {
  var j;
  j = i;
  return delay(0, function() {
    return console.log(j);
  });
};
for (i = 0; i <= 10; i++) {
  _fn();
}

As a bonus,

Strict guidelines will also be placed on internet providers which offer so-called unlimited packages, where users can download to their heart’s content. If a user incurs an “additional charge or suspension of service as a consequence of exceeding a usage threshold,” the ISP cannot use the term “unlimited”.

Finally! I hate it when providers are allowed to say ‘Unlimited’ and then place a limit on it (generally via a Fair Usage Policy). What a load of rubbish! I’m a heavy bandwidth user so it makes it very hard for me to find out what the limit is and then choose the relevant provider. Fortunately Virgin’s XXL broadband truly is unlimited - at least for now!

I was having an issue with Disqus - my comments were showing on post pages, but the comment count was not. Turns out that this is a simple to fix error relating to JavaScript scope.

Scope issue

The developer of Octopress has kindly protected our window object from being polluted with various unnecessary variables by wrapping the whole disqus include in an anonymous function.

However, the Disqus script actually injects another <script> element into the <head> (or <body>) of the page. This injected script, like all other scripts on the page, inherits the global (window) scope, and not the scope of this anonymous function. Thus it will not have access to the disqus variables such as disqus_shortname defined within the anonymous function. Without these variables, disqus loads such domains as undefined.disqus.com which is not very helpful!

Solution

The fix is simple - open up source/_includes/disqus.html and move the beginning of the anonymous function ((function () {) so that it does not contain the var disqus_* variable definitions. We also need to insert a couple of semi-colons that have been accidentally omitted. Here’s the resulting file:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

{% comment %} Load script if disquss comments are enabled and `page.comments` is either empty (index) or set to true {% endcomment %}
{% if site.disqus_short_name and page.comments != false %}
<script type="text/javascript">
      var disqus_shortname = '{{ site.disqus_short_name }}';
      {% if page.comments == true %}
        {% comment %} `page.comments` can be only be set to true on pages/posts, so we embed the comments here. {% endcomment %}
        // var disqus_developer = 1;
        var disqus_identifier = '{{ site.url }}{{ page.url }}';
        var disqus_url = '{{ site.url }}{{ page.url }}';
        var disqus_script = 'embed.js';
      {% else %}
        {% comment %} As `page.comments` is empty, we must be on the index page. {% endcomment %}
        var disqus_script = 'count.js';
      {% endif %}

    (function () {
      var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
      dsq.src = 'http://' + disqus_shortname + '.disqus.com/' + disqus_script;
      (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
    }());
</script>
{% endif %}

One more thing

If you want your links to be absolute, then you should set root to be your root URL in _config.yml - e.g. root: http://www.benjiegillam.com/ rather than root: /. This may or may not be required for Disqus to function properly.

EDIT: @Octopress has confirmed that root must not contain the scheme and host.

Bonus

Now that I’ve implemented the above, my comments and comment counts successfully display on my benjiegillam.dev site (hosted by Pow)!

Annoyingly, Mac OS X’s sed is not 100% compatible with GNU sed, and so this solution from StackOverflow did not immediately solve my issue:

Or use this solution with sed: sed ':a;N;$!ba;s/\n/ /g' This will read the whole file in a loop, then replaces the newline(s) with a space. Update: explanation.

1. create a register via :a
2. append the current and next line to the register via N
3. if we are before the last line, branch to the created register $!ba ($! means not to do it on the last line (as there should be one final newline)).
4. finally the substitution replaces every newline with a space on the pattern space (which is the contents of the a register = the whole file.

The solution I found was to split the command up into explicit separate commands:

1

sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/ /g'

After running exitwp to import my blog, I ran rake generate to build it, and got the following issue:

$ rake generate
(in /Users/benjiegillam/Documents/Blog/octopress)
## Generating Site with Jekyll
Configuration from /Users/benjiegillam/Documents/Blog/octopress/_config.yml
unchanged sass/screen.scss
Building site: source -> public
/Users/benjiegillam/Documents/Blog/octopress/plugins/raw.rb:11:in `gsub': invalid byte sequence in UTF-8 (ArgumentError)
    from /Users/benjiegillam/Documents/Blog/octopress/plugins/raw.rb:11:in `unwrap'
    [...]

However, converting the file was perfectly valid UTF-8 as confirmed by an iconv -c conversion followed by a diff -u.

After a quick bit of hacking in the octopress/plugins/raw.rb file to spit out the content that was being converted, I found the file at fault. After some iteration I got to the root of the issue - Octopress’ default markdown parser, rdiscount, REALLY doesn’t like UTF-8 characters in URLs. I’ve built a test here:

I had a few issues going from Wordpress to Octopress (which I did using exitwp) - I’ve outlined some of my solutions below.

Crossed out code blocks

The markdown produced by pandoc (an exitwp dependency) uses ~~~~ in places to denote a code block. However, rdiscount (Octopress’ markdown parser) recognises this instead as strike-through. A simple bash command fixes this (though it does lose some of the formatting details).

1
2
3
4
5

for I in *; do sed -i '' -e 's/^~~~~ {\(.*\)}$/\
<!-- \1 -->\
```/g' $I; done
for I in *; do sed -i '' -e 's/^~~~~$/\
```/g' $I; done

Line breaks in links

For some reason exitwp added some linebreaks into the markup for the links. And for some other reason, octopress’ style sheet (screen.css) tells <a> tags that their white-space should be pre-wrap, these two issues combine to give you broken links.

This next command should fix these newline issues.

If you’re trying to access a session from a socket.io socket, I recommend following these instructions which work great for MemoryStore. However for RedisStore you need to do a little more work because the object returned is just a JSON object - not a Session object as it should be. I’ve hacked around it by adding the following (CoffeeScript) to convert it to a Session object if required:

1
2
3
4
5

if session and not session.prototype?.reload
    req =
        sessionStore: sessionStore
        sessionID: data.sessionID
    session = new express.session.Session req, session

1
2
3
4
5
6
7
8
9

var req, session, _ref;

if (session && !((_ref = session.prototype) != null ? _ref.reload : void 0)) {
  req = {
    sessionStore: sessionStore,
    sessionID: data.sessionID
  };
  session = new express.session.Session(req, session);
}

immediately inside the sessionStore.get callback.

If you’re using nodemon to automatically restart your node/coffee server process when the source code changes (and if you’re not, why not?), you may find (as I did) that it can kill the script quite abruptly. If you want to do something when the code shuts down (close connections, flush to database/disk/redis/memcached, etc) then you can do so by intercepting the SIGUSR2 signal that nodemon uses to kill your script, and then setting up these actions. Once these actions are complete, you can exit but nodemon will see that as a crash and stop restarting. So you then have to kill your app again with SIGUSR2. Here’s the solution I use in CoffeeScript (click Toggle for JavaScript):

1
2
3
4
5
6
7
8
9
10
11
12

# Alias for setTimeout that reorders parameters to look neater in CoffeeScript
delay = (ms, cb) -> setTimeout cb, ms
# Handle SIGUSR2, but only once
process.once 'SIGUSR2', ->
  console.log 'Doing shutdown tasks...'
  delay 5000, ->
    console.log 'All done, exiting'
    # Kill ourself with the SIGUSR2 signal again
    process.kill process.pid, 'SIGUSR2'
# This next line just to prevent the app exiting in case you want to use this as a demo
delay 99999999, ->
  console.log 'App exiting naturally'

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

var delay;

delay = function(ms, cb) {
  return setTimeout(cb, ms);
};

process.once('SIGUSR2', function() {
  console.log('Doing shutdown tasks...');
  return delay(5000, function() {
    console.log('All done, exiting');
    return process.kill(process.pid, 'SIGUSR2');
  });
});

delay(99999999, function() {
  return console.log('App exiting naturally');
});

By the way, to install nodemon globally: sudo npm install -g nodemon

Update: Give up now. Skype never responds to the [SkypeAPI connect] method!

Original post follows: