valve.js

#

version 1.0

(c) 2011 Dennis Swennen
Valve is a JavaScript library for loading, manipulating and displaying data trough Yahoo(c) Pipes.
For all details and documentation: GitHub repository

#

Global variables

mainpipe is the global pipe object
wrapper contains information for the UI
pipeLoaded and pipeConfirmed are for usability in the UI, so that certain buttons only become available when e.g. the pipe is loaded
pipelist contains the current supported pipes from which the user can choose

var mainpipe,
	wrapper,
	pipeLoaded = false,
	pipeConfirmed = false,
	fetchIntervalId = 0,
	pipelist = {
		"ZKJobpaj3BGZOew9G8evXg" : {
			name : "Yahoo Finance Stock Quote",
			url : "http://pipes.yahoo.com/pipes/pipe.info?_id=ZKJobpaj3BGZOew9G8evXg",
			params : []
		},
		"c6884b96b6cf49c18ec7313b76ad6130" : {
			name : "Twitter Public Timeline",
			url : "http://pipes.yahoo.com/pipes/pipe.info?_id=c6884b96b6cf49c18ec7313b76ad6130",
			params : []
		},
		"OIa3h8_A3RGoQqRb_w6H4A" : {
			name : "Yahoo Finance Stock Quote Data",
			url : "http://pipes.yahoo.com/pipes/pipe.info?_id=OIa3h8_A3RGoQqRb_w6H4A",
			params : []
		},
		"7a31ec30311b205544ee744b872f4417" : {
			name : "Sparen en Beleggen - tijd.be",
			url : "http://pipes.yahoo.com/pipes/pipe.info?_id=7a31ec30311b205544ee744b872f4417",
			params : []
		},
		"f7ce5f42573f2aba3aa11fb99a38c66d" : {
			name : "Movie Database",
			url : "http://pipes.yahoo.com/pipes/pipe.info?_id=f7ce5f42573f2aba3aa11fb99a38c66d",
			params : []
		},
		"45ac0c6883a348080eaa4522ff10e9e7" : {
			name : "Combined Feeds",
			url : "http://pipes.yahoo.com/pipes/pipe.info?_id=45ac0c6883a348080eaa4522ff10e9e7",
			params : []
		},
		"vFM8Lf143BGYe6eEmLokhQ" : {
			name : "Search for Jobs: Source TeleportJobs",
			url : "http://pipes.yahoo.com/pipes/pipe.info?_id=vFM8Lf143BGYe6eEmLokhQ",
			params : []
		}
	};
	
#

Baseline setup for the Google Chart Visualisation API

google.load('visualization', '1', {
		'packages' : ['corechart', 'table', 'charteditor']
	});
#

Constructor functions

#

The pipe constructor, contains all the information for the loaded pipe
count the number of items in the pipe
value the original data from the pipe, an Array
params contains the original parameters for modified the pipe call
flatItems the items from the source, in a non-hierarchical Array
currentDataView is the current representation for the Google Chart visualisation

function pipe(json) {
	
	this.count = json.count;
	this.value = json.value; // original value parsed from the JSON strike
	this.gTable = new googleDataTable();
	this.params = {};
	this.flatItems = []; //contain all the items with flattened objects
	this.currentDataView = [];
	
}
#

The googleDataTable constructor, which contains a Google DataTable Object

function googleDataTable() {
	this.data = new google.visualization.DataTable();
	this.attributes = [];
}
#

GoogleDataTable Object prototype functions

#

Takes an JavaScript Object as parameter, makes it non-hierarchical and propagates the Google DataTable columns.

googleDataTable.prototype.setColumns = function (obj) {
	var flatObj = flatten(obj),
	clas;
	
	for (prop in flatObj) {
		clas = Object.prototype.toString.call(flatObj[prop]).slice(8, -1).toLowerCase();
#

check for "null" datatypes

		if (clas !== 'null' && clas !== 'undefined') {
			this.data.addColumn(clas, prop.toString(), "col_" + prop.toString());
			this.attributes.push(prop);
		}
	}
}
#

Input is an flat Array which contains all the items, each item is an object which is put in the Google DataTable

googleDataTable.prototype.fillColumns = function (dataArray) {
	var dataTable = this.data,
#

items = pipeObj.value.items,

	count = dataArray.length,
	i,
	propNr,
	tempObj;
	
#

first adds the empty rows in the DataTable

	dataTable.addRows(count); 
	
	for (i = 0; i < count; i++) {
		propNr = 0;
		tempObj = dataArray[i];
		for (prop in tempObj) {
			if (tempObj[prop] !== null && tempObj[prop] !== undefined) {
				dataTable.setCell(i, propNr, tempObj[prop]);
				propNr++;
			}
		}
	}
	
}
#

Pipe Object prototype functions

#

Converts from the mainpipe the orginal pipe items, which can be hierarchical format, into a flat Array for better processing.

pipe.prototype.flatten_data = function () {
	var pipeItems,
		pipeItemsLength;
	
	pipeItems = this.value.items;
	pipeItemsLength = pipeItems.length;
	
	for (i = 0; i < pipeItemsLength; i++) {
#

makes use of the flatten function

		this.flatItems.push(flatten(pipeItems[i]));
	}
	
}
#

to initialize the GoogleDataTable object

pipe.prototype.initDataTable = function () {
	this.gTable.setColumns(this.flatItems[0]);
	this.gTable.fillColumns(this.flatItems);
}
#

to update the GoogleDataTable object

pipe.prototype.updateGoogleData = function () {

	this.gTable = new googleDataTable();
	this.gTable.setColumns(this.flatItems[0]);
	this.gTable.fillColumns(this.flatItems);
	
}
#

User input the attribute name. The function returns an Array with all the values for the attribute, usefull to use the array of values as data somewhere else.

pipe.prototype.collect_data_array = function (key) {
	
	var arr = [],
	items,
	itemsLength;
	
	items = this.flatItems;
	itemsLength = this.flatItems.length;
	
	for (i = 0; i < itemsLength; i++) {
		arr.push(items[i][key]);
	}
	
	return arr;
}
#

Returns an object which contains all the pipe's parameters & their types

pipe.prototype.get_data_attributes = function () {
	
	var item,
	prop,
	type,
	attrListObj = {},
	i = 0;
	
	item = this.flatItems[0];
	
	for (prop in item) {
		type = Object.prototype.toString.call(item[prop]).slice(8, -1);
		attrListObj[prop] = {};
		attrListObj[prop].id = i;
		attrListObj[prop].type = type;
		attrListObj[prop].value = item[prop];
		i++;
	}
	
	return attrListObj;
}
#

USER Interface Section

#

Update the data in the pipe after e.g. converting the data.

function pipedata_update() {
	
#

update DATA

	var o;
	
	o = mainpipe.get_data_attributes();
	
	mainpipe.updateGoogleData();
	
#

update JSON view

	try {
		outputDoc = this.jsonFormatter.jsonToHTML(mainpipe.value, this.uri);
	} catch (e) {
		outputDoc = this.jsonFormatter.errorPage(e, this.data, this.uri);
	}
	
	document.getElementById('jsondata').innerHTML = outputDoc;
	addClickHandlers();
	
#

Create the table view in the UI

	hide_table_view();
	$('#DynamicGrid').append(CreateKeysView(o, "lightPro", true)).fadeIn();
	create_table_checkboxes();
}
#

Propagates the dropdownbox, where the user can choose a pipe in the UI, with al the `pipelist pipes.

function createPipeList() {
	var dropdownbox = document.getElementById('pipelist');
	
	for (var id in pipelist) {
		addOption(dropdownbox, pipelist[id].name, id);
	}
}
#

update the pipe parameter form in the UI

function update_parameters_form() {
	
	var id;
	
	if ($('#pipelist').innerHTML !== '')
		id = GetSelectedValue('pipelist');
	
#

resets the visibility of the buttons because a new pipe is loaded

	$('.button:not("#autofetchbutton"):not("#importpipebutton")').css('visibility', 'hidden');
	$('#div_timeline, #div_googlechart')
	.css('visibility', 'hidden')
	.css('display', 'none');
	createYQLscriptTag(id);
	
	$('#keytable').remove();
	pipeLoaded = false;
}
#

removes the data view table

function hide_table_view() {
	$('#keytable').remove();
}
#

Main Yahoo Pipe call & process functions

#

Uses the Yahoo Query Scripting language for fetching the parameters automatically from the pipe information page on the website of Yahoo and converts into the correct URL to load the pipe.

function createYQLscriptTag(pipeID) {
	
	var uri,
	scriptEl,
	scriptInsert;
	
	uri = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where%20url%3D%22http%3A%2F%2Fpipes.yahoo.com%2Fpipes%2Fpipe.info%3F_id%3D' + pipeID + '%22%20and%20xpath%20%3D%20\'%2F%2Fdiv%2Fform%2Ftable%2F*%2F*%2Finput%5Bnot%20(%40name%3D%22_cmd%22)%5D\'&format=json&diagnostics=true&callback=set_pipe_parameters';
	
	scriptEl = document.createElement('script');
#

scriptEl.type = 'text/javascript';

	scriptEl.src = uri;
	scriptInsert = document.getElementsByTagName('script')[0];
	scriptInsert.parentNode.insertBefore(scriptEl, scriptInsert);
	
#

console.log(uri);

	
}
#

Callback function where o contains the parameters for the loaded pipe. It creates the pipe parameter form so the user can modify the parameters for the pipe to load.

function set_pipe_parameters(o) {
	
	var list,
	id,
	el,
	out_div,
	text = "";
	
	out_div = document.getElementById("pipeparameters");
	text += '<fieldset>';
	
	if (o.query.count > 0) { // the pipe has parameters
		list = o.query.results.input;
		id = GetSelectedValue('pipelist');
		pipelist[id].params.length = 0; //clears the parameters for the current selected pipe
		
#

multiple returned parameters

		if ($.isArray(list)) {
			for (i = 0; i < list.length; i++) {
				pipelist[id].params.push(list[i]);
				text += "<p><label for='value1'>" + list[i].name + "</label><input type='text' class='required param_value' name='" + list[i].value + "' id='value" + (i + 1) + "' /></p>";
			}
		}
#

only found 1 parameter

		else { //Object
			pipelist[id].params.push(list);
			text += "<p><label for='value1'>" + list.name + "</label><input type='text' class='required param_value' name='" + list.value + "' id='value1'/></p>";
		}
		
	} else {
		text = "(This pipe has no parameters)";
	}
	
	text += "</fieldset>"
	out_div.innerHTML = text;
	
}
#

Creates the full URI for requesting the JSON data from Yahoo Pipes.

function createRequestURI(id) {
	
	var fullUri,
	paramObj,
	paramList,
	param_values,
	rendertype,
	uri,
	str = '';
	
	uri = pipelist[id].url;
	
#

get the parameters for the pipe from the user input fields

	paramList = pipelist[id].params;
	param_values = document.getElementsByClassName("param_value")
		
		for (i = 0; i < paramList.length; i++) {
			str += "&" + paramList[i].name + "=" + param_values[i].value;
		}
		
		fullUri = uri.replace("info", "run");
	fullUri += "&_render=json",
	fullUri += str;
	
	console.log( fullUri );
	return fullUri;
	
}
#

Loads the pipe, after creating the URL for the pipe call and waits for the response from the Yahoo Pipes server. Afterwards the response is converted into a pipe Object.

function pipeCallCORS() {
	
	var request,
	id,
	inputurl;
	
	id = GetSelectedValue('pipelist');
	
	if ($('input:text.param_value[value=""]').length > 0) {
		update_status_text("no parameter", "error");
	} else {
		
		request = createCORSRequest("get", createRequestURI(id));
		
		update_status_text("pipe is loading..", "warning");
		document.getElementById('statusicon').style.display = 'inline';
		
		if (request) {
			request.onload = function () {
#

the actual parsing from JSON string to a Javascript object, it uses the pipeReviver function to parse in the correct format.

				var r = JSON.parse(request.responseText, pipeReviver); 
#

If the request has a response r.

				if (r.count != 0) {
					update_status_text(r.count + " pipe items loaded");
					processJSONResponse(r);
					pipeLoaded = true;
					$(".button").css('visibility', 'visible');
					$("#stopfetchbutton").css('visibility', 'hidden');
				}
				
			};
			request.send();
		}
	}
	
}
#

Creates the Cross-Origin Resource Sharing request which is sent to the Yahoo Pipes server. The input is the url created by createRequestURI

function createCORSRequest(method, url) {
	update_status_text("fetching data..");
	var xhr = new XMLHttpRequest();
	if ("withCredentials" in xhr) {
		xhr.open(method, url, true);
	} else
		if (typeof XDomainRequest != "undefined") {
			xhr = new XDomainRequest();
			xhr.open(method, url);
		} else {
			xhr = null;
		}
	return xhr;
}
#

start the automatic processing of the selected pipe. The pipe is called and so the data is updated every 10 seconds. Starts when the user clicks the button in the UI to start the loading.

function start_feed() {
	update_status_text("auto fetch running");
	auto_feed();
	fetchIntervalId = window.setInterval( "auto_feed()", 10000 );
}
#

stops the auto-fetching for the current selected pipe

function stop_feed() {
	update_status_text("auto fetch stopped", "error");
	window.clearInterval(fetchIntervalId);
	document.getElementById("div_autofeed").innerHTML = "";
	document.getElementById('statusicon').style.display = 'none';
	$("#stopfetchbutton").css('visibility', 'hidden');
}
#

Function for fetching the data from the Yahoo Pipe and displaying it in the User Interface. This function is called when auto-fetch is enabled.

function auto_feed() {

	var request,
	renderType,
	id,
	inputurl,
	uri,
	container,
	feed,
	str, s, h, m;
	
	id = GetSelectedValue('pipelist');
	if ($('input:text.param_value[value=""]').length > 0) {
		update_status_text("no parameter", "error");
	} else {
		uri = createRequestURI(id);
		request = createCORSRequest("get", uri);
		document.getElementById('statusicon').style.display = 'inline';
		
		if (request) {
			request.onload = function () {
				var r = JSON.parse(request.responseText, pipeReviver); 
#

If the request has a response r.

				if (r.count != 0) {
					$("#stopfetchbutton").css('visibility', 'visible');
					
					mainpipe = new pipe(r);
					mainpipe.flatten_data();
					feed = mainpipe.flatItems;
					pipeLoaded = true;
					
					container = document.getElementById("div_autofeed");
					
					str = "<ul>";
					for(i=0; i < feed.length; i++) {
						s = feed[i].pubDate.getSeconds();
						h = feed[i].pubDate.getHours();
						m = feed[i].pubDate.getMinutes();
						str+= "<p><span class='pubDate'>"+h+":"+m+"</span>"+"<span class='title'>"+feed[i].title+"</span></p>";
					}
					container.innerHTML = str;
					update_status_text("feed updated");
				}
				
			};
			request.send();
		}
	}
	
}
#

Handles the Pipe response Callback. This function handles the loaded pipe data. It creates a new pipe Object, puts the fetched pipe items in the flatItems Array It also creates the table view in the UI and the details JSON view.

function processJSONResponse(response) {
	
	var o;
	
	mainpipe = new pipe(response);
	mainpipe.flatten_data();
	mainpipe.initDataTable();
	o = mainpipe.get_data_attributes();
	
#

The view is created from the first pipe item to show all the attributes for an item.

	$('#keytable').remove();
	$('#DynamicGrid').append(CreateKeysView(o, "lightPro", true)).fadeIn();
	create_table_checkboxes();
	
#

the detailed JSON view

	this.jsonFormatter = new JSONFormatter();
	
	try {
		outputDoc = this.jsonFormatter.jsonToHTML(mainpipe.value, this.uri);
	} catch (e) {
		outputDoc = this.jsonFormatter.errorPage(e, this.data, this.uri);
	}
	
	document.getElementById('jsondata').innerHTML = outputDoc;
	addClickHandlers();
	
	document.getElementById('statusicon').style.display = 'none'; //remove loading indicator
	
}
#

Visualisation functions.

#

gets the selected attributes from the displayed data table. the attributes name and key are stored in the currentDataView object. The key is the number used for the Google DataTable columns.

function get_selected_attributes() {
	
	var selectedNames,
	selectedValues,
	key,
	value,
	i,
	hash = [], //result
	div,
	str,
	count = 0;
	
	selectedNames = $("#keytable").find(".selected th span");
	selectedValues = $("#keytable tr")
		.filter(".selected")
		.find("td.numberCell");
	
	for (i = 0; i < selectedNames.length; i++) {
		key = selectedNames[i].innerHTML;
		value = parseInt(selectedValues[i].innerHTML, 10);
		hash[value] = key;
	}
	
	div = document.getElementById("selectedattr_div");
	
	str = '<ul>'
		
		for (i = 1; i < hash.length; i++) {
			str += "<li>Column " + i + ": " + hash[i] + "</li>";
			count++;
		}
		
#

stores the selected attributes in the main pipe

		mainpipe.currentDataView = hash; 
	
#

update User interface

	div.innerHTML = str;
	update_status_text(count + " attributes selected");
	pipeConfirmed = true;
	
}
#

draws the Google visualisation from the data and the selected attributes. The visualisation type is retrieved from the User Interface.

function draw_selected_attributes() {
	
	var dataTable,
	dataView,
	chart,
	colNumbers = [],
	colTypes = [],
	keyNamesArray = [],
	chart = 'ColumnChart',
	prop,
	dataAttributes,
	i,
	n,
	a,
	chartEditor = null,
	typeChart,
	container,
	components,
	valid = false;
	
#

if a pipe is loaded and if the attributes to visualise are confirmed

	if (pipeLoaded && pipeConfirmed) {
		
		var div = $('#div_googlechart');
		div.css('display', 'inherit').css('visibility', 'visible');
		
		dataAttributes = mainpipe.get_data_attributes();
		a = mainpipe.currentDataView;
		
#

get the chart type from the user interface

		typeChart = GetSelectedText('typeChart');
		
		
#

pushes the key from the selected attributes in the Array colTypes

		for (i = 1; i < a.length; i++) {
			colTypes.push(dataAttributes[a[i]].type);
			colNumbers.push(dataAttributes[a[i]].id);
		}
		
#

Checks colTypes : if the chosen attributes match the type of the columns that is needed for the selected visualisation

		valid = validate_selected_types(typeChart, colTypes);
		if (valid) {
			dataTable = mainpipe.gTable.data;
			dataView = new google.visualization.DataView(dataTable);
			dataView.setColumns(colNumbers);
			
#

options for the Google chart

			wrapper = new google.visualization.ChartWrapper({
						chartType : typeChart,
						options : {
							width : 900,
							heigth : 300,
							showTip : true,
							enableScrollWheel : true,
							legend : "bottom"
						},
						containerId : 'visualization',
					});
			
#

draw the visualisation

			wrapper.setDataTable(dataView);
			wrapper.draw();
			
#

toolbox for exporting the chart into another website

			container = document.getElementById('googletoolbar_div');
			components = [{
					type : 'html',
					datasource : dataView.toDataTable().toJSON()
				}, {
					type : 'htmlcode',
					datasource : dataView,
					gadget : 'https://www.google.com/ig/modules/pie-chart.xml',
					userprefs : {
						'3d' : 1
					},
					style : 'width: 800px; height: 700px; border: 3px solid purple;'
				}
			];
			google.visualization.drawToolbar(container, components);
			update_status_text("Visualisation complete.");
		} else {
			update_status_text("Types not valid for the chosen visualisation", "error");
		}
	} else {
		update_status_text("Please confirm attribute selection.", "error");
	}
	
}
#

Various utility functions

#

Prints out debug variable in the debug div.

function debug(variable) {
	var outputdiv = document.getElementById('debug');
	if (variable === null) {
		outputdiv.innerHTML = "Variable undefined"
			
	} else {
		outputdiv.innerHTML += "<br />DEBUG:" + variable;
	}
}
#

Checks the type of the input obj and returns the Object class name a String

function typeCheck(type, obj) {
	var clas;
	
	clas = Object.prototype.toString.call(obj).slice(8, -1);
	return obj !== undefined && obj !== null && clas === type;
}
#

Converts the value of an Object's key into the type of the given type argument

function typeConvert(type, obj) {
	
	if ('Boolean' == type) {
		return (obj == "true" ? true : false);
	}
	
	if ('String' == type) {
		return obj = String(obj);
	}
	
	if ('Number' == type) {
		return parseInt(obj, 10);
	}
	
}
#

converts the attributes which are selected in the data view, into the type of choice. The type is chosen in the user interface.

function convert_selected_attributes() {
	
	var items,
	keysArray,
	key,
	type,
	i,
	outputDoc,
	div;
	
	items = mainpipe.flatItems;
	
#

get the attributes that need to be converted

	keys = $("#keytable").find(".selected th span");
	
	console.log(keys);
	
#

get the type

	type = GetSelectedText('conversionType');
	
	for (i = 0; i < items.length; i++) {
		for (j = 0; j < keys.length; j++) {
			key = keys[j].innerHTML;
			items[i][key] = typeConvert(type, items[i][key]);
		}
	}
	
	
#

replaces the modified items in the mainpipe

	mainpipe.flatItems = items;
	update_status_text(keys.length + " attribute(s) converted to " + type, "warning");
	div = document.getElementById("selectedattr_div").innerHTML = "";
	pipeConfirmed = false;
	pipedata_update();
	
}
#

Examines the JSON String variable when this is received from the pipe call and converts the String into the right JavaScript Object, including correct data types for certains known attributes.

function pipeReviver(key, value) {
	
	var d = new Date(),
	numbers = ['count', 'milliseconds', 'year', 'month', 'day', 'hour', 'minute', 'second', 'millisecond', 'day_of_week'];
	
	if ("object" === typeof value) {
		return value;
	} else if (numbers.indexOf(key) >= 0) {
		return +value;
	} else if (key === "pubDate") {
		return new Date(value);
	}
	
	return value;
}
#

Creates a linear, unhierarchical object from a structered object. This function is used for converting the loaded items from the pipe.

function flatten(obj) {
	var output = {},
	walk = function (j) {
		var jp;
		for (var prop in j) {
			jp = j[prop];
			if (jp == null) {
				output[prop] = jp;
			} else {
				if (jp.toString() === "[object Object]") {
					walk(jp);
				} else {
					output[prop] = jp;
				}
			}
		}
	};
	walk(obj);
	return output;
}
#

Helper function to check if the user's chosen attributes are suitable for the visualisation

function validate_selected_types(chartType,colTypes){

	var b = [],
		result,
		i;
	
	if( "BarChart" == chartType ) { //column 1 string, column 2 number
		b[0] = (colTypes[0] == "String");
		b[1] = (colTypes[1] == "Number");
	}
	
	if( "ColumnChart" == chartType ) {
		b[0] = (colTypes[0] == "String");
		for(i=1; i < colTypes.length; i++) {
			b[i] = (colTypes[i] == "Number");
		}
	}
	
	if( "PieChart" == chartType ) {
		b[0] = (colTypes[0] == "String");
		b[1] = (colTypes[1] == "Number");
	}
	
	if( "ScatterChart" == chartType ) { //all columns must be numbers
		for(i=0; i < colTypes.length; i++) {
			b[i] = (colTypes[i] == "Number");
		}
	}
	
	if( "LineChart" == chartType ) {
		b[0] = (colTypes[0] == "String");
		for(i=1; i < colTypes.length; i++) {
			b[i] = (colTypes[i] == "Number");
		}
	}
	
	if( "Map" == chartType ) {
		b[0] = (colTypes[0] == "Number");
		b[1] = (colTypes[1] == "Number");
		b[2] = (colTypes[2] == "String");
	}
	
	for(i=0; i < b.length; i++) {
		result = b[i];
	}
	
	return result;
	
}