Doing is the best form of learning

REST.  It's one of those things that most of us developers typically deal with on the client side of things, or if we deal with them on the server side, we're using an already written API which "REST-ifies" our data. It was somewhat of a mystery to me, and a challenge, to know how to implement an abstract RESTful interface. I had written the webserver I use in node.js which supports all kinds of things, but hadn't yet added REST support. Until last night!

It wasn't too much of an undertaking, I guess I implemented the webserver in such a way that could be easily extended with REST. That's good, go "past Jason".

Basically the server gets the request, attempts to rewrite the URL given the site's configuration of URL Rewrites (which I wrote), then it gets the content (server side processing), and writes the content on the response stream, along with doing cookie and header processing, g-zipping, cache headers, etc. Nearly everything a mature web server should do.  It's been around since March 2011 and I always go back to making it better.

I started adding REST to it last night around 10pm, and by 12:30 I was writing code against it using AngularJS.  The REST format is inspired by AngularJS, in that you can provide parameters expected in the format :id   (colon - name).

I always start with how I want to write code.  This, I find, is a very important aspect of how I architect things.  I don't want to write a whole bunch of code that I'll have to write each time I want to use the new feature I am adding. So I START with the code that I'll be writing to use the new feature. If this hasn't yet sunk in as to how important I think this is, let me add this sentence... There. It's very important to me and how I architect.

I know the architecture of my web server, and know that you can't just reference things inside of it that haven't been given a public interface. It really has no public interface. The web applications implement the interface to work within the web server. I am looking at a way to decouple them but for now, this is how it is. The "params" array was added in that way because of this.

this.get = {
    path: "/:id",
    handler: function(db, id, qs, callback){
            engine.getObject(db, id, callback);
        },
    method: "GET",
    params: [
        function(context){ return context.db; },
        "id"
    ]
}

 

So in my site code, I'm registering a "get" method that takes the id of an object, looks in the database, and returns it. A URL for the call may look like this: /resource/objects/53f810db8a8cda084e000001

Here's some node.js code that comes from my "resource" module. Within the resource module, you register resources and later check the URL and http method against the current resources, to see if this is a resource / REST call.

this.registerResource = function(domain, name, resource){
    var r = /:([a-zA-Z0-9]+)/g;    
    var m = null;
    for (var i in resource){
        var res = resource[i];
        var regexString = globalPart + name + res.path;
        while ((m = r.exec(res.path))!=null){
            regexString = regexString.replace(":" + m[1], "(.*?)");
        }
        var local = { domain: domain, name: name, method: res.method, regex: new RegExp(regexString), handler: res.handler, path: res.path, params: res.params };
        resources.push(local);
    }
}

For the resource, for which the above "get" code is just one method on a resource, find all the methods, replace the URL with a regex. Instead of the "path" which would be "objects/:id", it creates "/resource/objects/(.*?)", stores the original path, the method to handle it, the http request method (GET, POST, PUT, DELETE currently supported), and the params array.

When a request is made, find the resource with the following code, if no resource is found, or no resources on the current domain that match the HTTP Method, it's a standard request.

this.getResource = function(domain, url, method){
    var domainResources = resources.filter(function(r){ return r.domain == domain && r.method == method });
    if (domainResources.length == 0) return null;
    var resource = null;
    domainResources.forEach(function(res){
        if (url == globalPart + res.name + res.path) resource = res;  // prefer exact matches first
        else if (resource == null && res.regex.test(url)){
            resource = res;
        }
    })

    return resource;
}

The next methods call the resource handler. For GET / DELETE calls, the requestData is just the querystring, for PUT / POST calls, this will be the form data as parsed by the POST parser in node.

this.extractParamMap = function(url, resourcePath){
    var m = null, map = {};
    var urlParts = url.split("/");
    var resParts = resourcePath.split("/");
    for (var i = 0; i < resParts.length; i++){
        if (resParts[i].indexOf(":") == 0){
            map[resParts[i].substring(1)] = urlParts[i];
        }
    }
    return map;
}

this.handleResource = function(resource, url, context, requestData, callback){
    var params = [];
    var paramMap = this.extractParamMap(url, globalPart + resource.name + resource.path);

    for (var i = 0; i < resource.params.length; i++){
        if (typeof(resource.params[i]) == "function"){
            params.push(resource.params[i](context));
        }
        else if (typeof(resource.params[i]) == "string"){
            params.push(paramMap[resource.params[i]]);
        }
    }

    params.push(requestData);
    params.push(callback);

    resource.handler.apply(resource, params);
}

The code within the webserver which was modified to process resources looks like this.  Determine if it's a resource or standard request. Call accordingly.

        if (req.method == "POST" || req.method == "PUT"){
            post.parseForm(req, function(formData){
                if (loadedResource != null){
                    resource.handleResource(loadedResource, url, site, formData, function(data){
                        var content = {};
                        content.contentType = "application/json";
                        content.content = JSON.stringify(data);
                        finishedCallback(content);
                    });
                }
                else if (handler != null){
                    query.form = formData;
                    handler.handlePost(path, query, site, req, function(content){
                        if (jsonRequest) content.contentType = "application/json";
                        finishedCallback(content);
                    });
                }
            })
        }
        else if (req.method == "GET" || req.method == "DELETE"){
            if (loadedResource != null){
                resource.handleResource(loadedResource, url, site, query.querystring, function(data){
                    var content = {};
                    content.contentType = "application/json";
                    content.content = JSON.stringify(data);
                    finishedCallback(content);
                })
            }
            else if (handler != null){
                handler.handle(path, query, site, null, req, function(content){
                    if (jsonRequest) content.contentType = "application/json";
                    finishedCallback(content);
                });
            }
        }
        else {
            finishedCallback({contentType: "text/html", content: url + " - No handler found", statusCode: 404});
        }

In AngularJS, with the $resource module, this is cake.

        return $resource("/resource/objects/:id", {}, 
            { 
                list: { method: "GET", isArray: true },
                get: { method: "GET" },
                save: { method: "POST" },
                update: { method: "PUT" },
                remove: { method: "DELETE" }
            }
        );

 

That's it!  Later on I might find I need other things, but that's all the code that was required for now for handling resource / REST style methods. I will have a site up in a few weeks / months that will use this heavily. Then you can see it in action!

Fantasy Golf Tracking with Node.js, MongoDB, AngularJS, and Bootstrap - Part 2

I'll address this in parts since the first post really didn't cover anything technical, and was just a bunch of screenshots.

MongoDB Implementation

I previously went over the basic data structure for this app in the previous post.  Teams consist of the team name. Tournaments consist of the key, the name, the start and end date, the course name, the par for the course, and the current round. Also, bools for finished and in progress. Then there is the teams players per tournament which I called teamTournament. It also keeps historical records of their total for that tournament. This lead me to be able to build a leaderboard widget, since the lowest score at the end of the season gets a prize.

Tournament Sample Data

> db.tournaments.find().pretty()
{
        "_id" : ObjectId("53a0964d4fcdf5c39c912acd"),
        "key" : "us-open-2014",
        "name" : "U.S. Open",
        "scoresLocation" : "http://www.pgatour.com/data/r/026/leaderboard-v2.json",
        "startDate" : ISODate("2014-06-12T07:00:00Z"),
        "endDate" : ISODate("2014-06-15T19:00:00Z"),
        "latestRound" : 4,
        "inProgress" : false,
        "isFinished" : true,
        "course" : "Pinehurst No. 2",
        "par" : 70
}

Granted, that inProgress and isFinished can be determined real time, but that's ok.

Team Tournament Sample Data

> db.teamTournament.find().pretty()
{
        "_id" : ObjectId("53a2e61ed42498e823000001"),
        "players" : [
                "28259",
                "28087",
                "21209",
                "08075",
                "31202",
                "28486"
        ],
        "teamId" : ObjectId("53a193fc4fcdf5c39c912af7"),
        "tournamentId" : ObjectId("53a20488d7aee2e01b000001"),
        "tournamentTotal" : 1073
}

I used to use DBRef for referencing other collections, but then I found out that unless you don't know at runtime, you should just use ObjectID.  So my ObjectID of 53a19 etc is in the field of teamId, so I know it's a team reference. If it were called "documentId" and I had collections of "images" and "html snippets" and "swf files", then I could use the DBRef, since it could be one of 3 different collections I want to reference. But since I know, MongoDB says it's much more efficient to just use ObjectID references.

Node.js to Access the Database

As for writing code to access this, I use the Node MongoDB Native library for accessing node, and a helper class that I wrote so I'm not writing lots of code to do things that I do frequently. For instance, my code for getting tournaments looks like this:

this.getTournament = function(db, key, callback){
    dbhelp.findOne(db, "tournaments", { key: key }, function(tournament){
        callback(tournament);
    });
}

this.getCurrentTournament = function(db, callback){
    var upcoming = new Date().addDays(3);
    var past = new Date();

    dbhelp.find(db, "tournaments", { "$or": 
            [ 
                { "startDate": { "$lt": upcoming } },  
                { "endDate": { "$lt": past } }
            ] 
        }, null, { startDate: 1 }, function(tournaments){
        if (tournaments != null && tournaments.length > 0){ return callback(tournaments[0]); }
        else return callback(null);
    });
}

this.getTournaments = function(db, year, callback){
    var start = new Date(year, 0, 1);
    var end = new Date(year, 11, 31);

    dbhelp.find(db, "tournaments", { startDate: { "$gt": start }, endDate: { "$lt": end } }, function(tournaments){
        callback(tournaments);
    });
}

So, you can see, not a lot of code for getting a specific tournament, getting the current tournament, and getting all tournaments this year. (Also I've modified the Date prototype to include a .addDays method).

That is some sample code that I've written for this application. It covers the back end. Next up I'll cover the front end, using AngularJS and Bootstrap to make it work well and look great!

Fantasy Golf Tracking with Node.js, MongoDB, AngularJS, and Bootstrap - Part 1

My family and a bunch of friends are in a fantasy golf league together. The rules are pretty straightforward, although probably not standard.

Rules:
1. Pay $50
2. 10 Tournaments that span the PGA Tour season.
3. Pick 6 golfers before the first tee times on Thursday typically.
4. 4 lowest scores are added and that's your score for each day.
5. If 3 of your players miss the cut, you are assigned the worst score at the end of round 3, pretty much destroying your chance to win.
6. Lowest score wins. $50 payoff for majors (Masters, US Open, British Open, PGA Championship), $25 for the other tournaments.

My brother Pat is the score keeper and chairman of the league. The data collection and reporting was pretty much done in Excel.  This is a fine method for doing such things. The scores would be emailed out along with entertaining commentary.

But then it was my turn to do the data since Pat was going to Boston for the US Open weekend to visit some friends.

Excel was not a viable solution.

I happened to stumble on the PGA Tour Leaderboard website. I noticed that the data is loading in asynchronously, which could only mean AJAX. The question was, which format, and what data is available?

The answer was the start of this application. The data is JSON. And EVERYTHING is available. (I hope I don't get in too much trouble for grabbing it :). Well, everything I needed and some extra stuff.

The first step to building this app was to determine what the Information Architecture was going to look like.  Here's what I came up with:

Teams: Name, Email address of a person in the league.
Tournaments:  Tournament info include URL to the JSON file on pgatour.com, data like start / end date, is it in progress, is it finished.
Team Tournament: Each team's 6 initial golfers for each tournament, and total score recording.

Pulling in the tournament from the pgatour.com JSON file pulls in all of the information required for a tournament in my system, so all that is needed as input is the JSON file URL!

Next you assign teams. There can be 6 max.

Then scores are calculated.

And updated throughout the round, each day, for the whole tournament.

If a player doesn't have 4 players make the cut, they are given a substitute. The worst score from the 3rd round.

That is pretty much everything!  Once again, working with AngularJS is a breeze.

Check out the site at fantasygolf.jasontconnell.com! There you can view the source to get an idea of how it was written. It was very simple that I don't even feel like I can post any difficult code that I wrote, but it's just that it's really cool. Our next tournament is the Congressional next week, I hope everyone in the league uses the site.  I added GA tracking so I'll know for sure :)

Javascript Functional Programming

In work, I was recently tasked with a data comparison project. A bunch of data was getting duplicated. These records include user first and last names, their phone, email, company.  The master file was a list of these same users but they were assigned "card id" because they were already customers.

The method was to find a match for name (first and last) and if the email didn't match, then separate it out. Then find a match for email, and if the name didn't match, separate it out.

You can imagine this logic:

Go through each line... 
var matches = core.filter(function(a){ return a.firstName == line.firstName && a.lastName == line.lastName; }); // match names 
if (matches.length == 1){ var matchingEmail = matches.filter(function(a){ return a.email == line.email; });
    if (matchingEmail.length == 1){ add it to results }
     else separate it out;
}

And do the same, checking email first. The main feature of Object Oriented Programming is to abstract out data and functionality so that different objects can behave the same way in a system. The main idea of functional programming is that the process can be abstracted out.

What I'm basically doing is, match the line to the map based on some values, if there are any matches, find matches within that result which match some other value. If that result is 1, then we've found a good match, and repeat with the comparison methods swapped.

Or, even more abstractly... running a comparison function, getting results, and running another comparison function to filter the results further.

Let's define our comparison functions:

var compareName = function(a) { return a.firstName == line.firstName && a.lastName == line.lastName; };
var compareEmail = function(a) { return a.email == line.email; };

We can create a method to run these comparisons. It will need the line, the master list, and the two comparison functions.

function compare(line, masterList, comparison1, comparison2)

And then we can call it like so:

compare(line, masterList, compareName, compareEmail);
compare(line, masterList, compareEmail, compareName);

Voila!

I should write more Node.js posts

They are consistently my top rated posts from an SEO standpoint. Also, according to Google Analytics, they lead to an average of a good 5 minutes on the site.  I will come up with more, but I feel like I'm not doing anything noteworthy or extraordinary with Node.js at the moment. I've put a lot of time and coding into the foundation of my site and other sites I've written, and I've already written a lot about building up that foundation.  I will come up with some other stuff to write about, it's what the people want!

Fixed!

The latest update to run the webserver with nohup seems to have fixed my server issues. I did get a server crash soon after changing the script, but I couldn't track it down, but it's been running non-stop for a few days now.

Finally I can put this to rest. Also, out of it, I got more efficient tracking as well as more detailed error messages in some instances, and less detailed error messages in others (for files not found, for instance, it just spits out 'Not found: ' and the file name, instead of a whole stack trace.).

Moment of Truth #3

After adding some better logging and stuff in the webserver, I determined that no error was causing the server to close. While I was starting the service with node webserver &, I think that I still needed to do nohup.

We shall see. My nohup'ed webserver is now running, and will run always in the future, whether it dies randomly or not. It's just the better way to start it.

Moment of Truth #2

After upgrading to Node.js 0.10.21 and implementing domains in Node.js, I was still getting errors. Since then, I've added code to listen for the response 'close' event, as well as the server 'clientError' event, in an effort to track down the issue. I think I may be on to something. At least then I'll be able to go to stack overflow or github with the issue in hopes that it will be fixed, or the act of listening for those events and logging the errors will lead me to the fix myself. Here's hoping.

The Moment of Truth

Every morning I have an email from pingdom that my site went down overnight. I've upgraded to node.js 0.10.20 a few weeks ago to take advantage of some other bug fixes and optimizations, and it shows in the Google Analytics page load times (they're all 0 seconds)

I have had no luck tracking down the cause, but I read about how to prevent a server error from bringing down the site. I've implemented the suggested solution using node.js domains, and we will see what's happening tomorrow.

It might be the same thing because when I go into the server, node is still running my webserver, it just appears the socket was destroyed... We shall see. Wish me luck!

Flickr Integration Complete!

It really didn't take too long! What I outlined in the previous post is exactly what it does:

  • Check flickr for my photos that were tagged with "jtccom". Download the 2048 pixel wide version of it
  • Write JS + Node part of my website which shows, in a queue like manner, an image that hasn't been processed
  • Clicking on the image shows a box of where I clicked, where it would be cropped, according to the sizes defined for the header
  • Send request to the server, which uses GraphicsMagick to crop it and then create different scaled images based on the sizes defined for the site

Here are some screenshots:

At first, you are prompted with the next image which hasn't been processed


Next, you select the point where you want to crop it. Sizes are pre-determined, so there's no dragging and resizing a bounding box, it knows all the sizes and the size of the images, so it just does it for you


Click the process button when you've made your crop selection


Wait a second or two while Node.js and gm (GraphicsMagick) processes your photos.

And GraphicsMagick code in node.js, which is really helpful, and I was able to get it to work on Windows

this.handlePost = function(site, query, finishedCallback){ var tmpdir = path.normalize(site.path + site.config.tempDownloadFolder); var processeddir = path.normalize(site.path + site.config.processedFolder); var form = query.form; var sizes = site.config.imageWidths; var heights = site.config.imageHeights; var filename = form.image.substring(form.image.lastIndexOf("/")+1); var fileParts = filename.split("."); sizes.sort(function(a,b){ return b - a; }); heights.sort(function(a,b){ return b - a; }); var x1 = parseInt(form.x1, 10), x2 = parseInt(form.x2, 10), y1 = parseInt(form.y1, 10), y2 = parseInt(form.y2, 10); var w = x2 - x1, h = y2 - y1; // process first size, use that for base of resizes var croppedPath = processeddir + fileParts[0] + "-" + sizes[0] + "." + fileParts[1]; gm(tmpdir + filename).crop(w, h, x1, y1).write(croppedPath, function(err){ var sync = new SyncArray(sizes); sync.forEach(function(size, index, array, finishedOne){ if (index > 0){ var scaled = processeddir + fileParts[0] + "-" + size + "." + fileParts[1]; gm(croppedPath).resize(size, heights[index]).write(scaled, function(x){ finishedOne(); }); } else finishedOne(); }, function(){ finishedCallback({ content: JSON.stringify({ success: true }), headers: {"Content-Type": "application/json"} }); }); }); }

Again, it uses my custom built webserver and the SyncArray object that I also wrote.

Flickr code was pretty simple too. Here's that, accessing the Flickr API (no auth) with Node.js

var http = require("http"), querystring = require("querystring"), SyncArray = require("syncarray").SyncArray; this.getPhotosByTag = function(apiKey, user, tag, callback){ var self = this; var method = "flickr.photos.search"; var qs = { method: method, api_key: apiKey, user_id: user, tags: tag, format: "json", nojsoncallback: 1 }; var req = { host: "api.flickr.com", path: "/services/rest/?" + querystring.stringify(qs) }; http.get(req, function(res){ var json = ""; res.on("data", function(d){ json += d; }).on("end", function(){ var photos = JSON.parse(json).photos.photo; if (photos.length > 0){ var sync = new SyncArray(photos); sync.forEach(function(photo, index, array, finishedOne){ self.getPhotoSizes(apiKey, photo.id, function(sizes){ photo.url = sizes.filter(function(s){ return s.label == "Large 2048"; })[0].source; finishedOne(); }) }, function(){ console.log("url = " + photos[0].url) callback(photos); }) } else callback([]); }); }); } this.getPhotoSizes = function(apiKey, photoId, callback){ var method = "flickr.photos.getSizes"; var qs = { method: method, api_key: apiKey, photo_id: photoId, format: "json", nojsoncallback: 1 }; var req = { host: "api.flickr.com", path: "/services/rest/?" + querystring.stringify(qs) }; http.get(req, function(res){ res.setEncoding("utf8"); var json = ""; res.on("data", function(d){ json += d; }).on("end", function(){ var sizes = JSON.parse(json).sizes.size; console.log(sizes.length); callback(sizes); }); }); }

The next step is to update the front end css to include all sizes of a version of the image, and switch between them using the respond.js and media queries. That should be simple, but it's late and I'm going to bed!!

Enjoy! Leave a comment.