Merge branch 'testing' into master

Started working on unit tests,
also cleaned up some of the sql row iteration.

spec/home_spec.lua

@@ -3,23 +3,24 @@ Test the home page
 ]]
 
 
-
+--[[
-
-describe("smr",function()
-	describe("site home page",function()
-		it("detours configure",function()
-			local s = {}
-			local c = false
 function configure(...)
 	local args = {...}
 	if args[1] == s then
 		c = true
 	end
 end
+]]
+
+describe("smr",function()
+	describe("site home page",function()
+		it("detours configure",function()
+			local s = {}
+			local c = false
 			local oldconfigure = configure
-			local index_get = require("index_get")
+			--local index_get = require("endpoints.index_get")
-			configure(s)
+			--configure(s)
-			assert(c)
+			--assert(c)
 		end)
 	end)
 	describe("author home page",function()

spec/pages_sanity_spec.lua

@@ -1,3 +1,4 @@
+function() return math.random() > 0.5 and "plain" or "imageboard" end
 
 local pages = {
 	index = {
@@ -12,15 +13,29 @@ local pages = {
 		name = "post_story",
 		methods = {
 			GET={},
-			POST={}
+			POST={
+				title = rng_any,
+				text = rng_any,
+				pasteas = rng_subdomain,
+				markup = rng_markup
+				tags = rng_any;
+			}
 		}
 	},
 	edit = {
 		route = "/_edit",
 		name = "edit",
 		methods = {
-			GET={},
+			GET={
-			POST={},
+				story=rng_storyid
+			},
+			POST={
+				title = rng_any,
+				text = rng_any,
+				pasteas = rng_subdomain,
+				markup = rng_markup,
+				tags = rng_any
+			},
 		}
 	},
 	--TODO:bio
@@ -29,7 +44,10 @@ local pages = {
 		name = "login",
 		methods = {
 			GET={},
-			POST={},
+			POST={
+				user = rng_subdomain,
+				pass = rng_any
+			},
 		}
 	},
 	claim = {
@@ -37,14 +55,18 @@ local pages = {
 		name = "claim",
 		methods = {
 			GET = {},
-			POST = {}
+			POST = {
+				user = rng_subdomain
+			}
 		}
 	},
 	download = {
 		route = "/_download",
 		name = "download",
 		methods = {
-			GET = {},
+			GET = {
+				story = rng_storyid
+			},
 		}
 	},
 	preview = {
@@ -64,10 +86,11 @@ local pages = {
 	
 }
 
-local request_stub_m = {
+local request_stub_m = {}
-}
 function http_response(req,errcode,str)
-	s = true
+	req.responded = true
+	req.errcode = errcode
+	req.message = str
 end
 function http_request_get_host(reqstub)
 	return "localhost:8888"
@@ -76,6 +99,16 @@ function http_request_populate_post(reqstub)
 	reqstub.post_populated = true
 end
 
+local function fuzz_endpoint(endpoint, parameters)
+	for i = 1,100 do
+		local req = {}
+		for paramtype, params in pairs(parameters) do
+
+		end
+	end
+	return true
+end
+
 describe("smr",function()
 	for name, obj in pairs(pages) do
 		describe("endpoint " .. name,function()
@@ -83,20 +116,19 @@ describe("smr",function()
 				describe("method " .. method,function()
 					local fname = string.format("%s_%s",name,string.lower(method))
 					it("should be named appropriately",function()
-						local f = assert(io.open(fname .. ".lua","r"))
+						local f = assert(io.open("endpoints/"..fname .. ".lua","r"))
 					end)
 					it("should run without errors",function()
-						require(fname)
+						require("endpoints." .. fname)
 					end)
 					it("should return a function",function()
-						local pagefunc = assert(require(fname))
+						function configure(...) print("configure called") end
+						local pagefunc = assert(require("endpoints." .. fname))
 						assert(type(pagefunc) == "function")
 					end)
-					it("calls http_response()",function()
+					it("should call http_response() at some point",function()
-						local pagefunc = require(fname)
+						local pagefunc = require("endpoints." .. fname)
-						local s = false
+						assert(fuzz_endpoint(pagefunc,parameters))
-						local reqstub = {}
-						pagefunc(reqstub)
 					end)
 
 				end)

spec/parser_imageboard_spec.lua

@@ -0,0 +1,12 @@
+
+describe("smr imageboard parser",function()
+	it("should load without error",function()
+		local parser = require("parser_imageboard")
+	end)
+	it("should accept a string and return a string",function()
+		local parser = require("parser_imageboard")
+		local input = "Hello, world!"
+		local output = parser(input)
+		assert(type(output) == "str
+	end)
+end)

src/libkore.c

@@ -256,7 +256,7 @@ lhttp_file_get(lua_State *L){
 }
 
 /*
-log(priority,string) //formating must be done before calling
+log(priority::integer,message::string) //formating must be done before calling
 */
 int
 lkore_log(lua_State *L){

src/lua/cache.lua

@@ -1,3 +1,10 @@
+--[[
+Implements a simple in-memory cache. The cache has no upper size limit, and
+may cause out-of-memory errors. When this happens, the OS will kill the kore
+worker process, and the kore parent process will restart with a fresh, empty
+cache
+]]
+
 local sql = require("lsqlite3")
 
 local queries = require("queries")
@@ -52,7 +59,6 @@ function ret.render(pagename,callback)
 		stmnt_cache:reset()
 		--page is not cached
 	elseif err == sql.ROW then
-		print("Cache hit:" .. pagename)
 		data = stmnt_cache:get_values()
 		stmnt_cache:reset()
 		return data[1]
@@ -75,7 +81,6 @@ function ret.render(pagename,callback)
 end
 
 function ret.dirty(url)
-	print("Dirtying cache:",url)
 	stmnt_dirty_cache:bind_names{
 		path = url
 	}

src/lua/config.lua

@@ -1,4 +1,7 @@
-
+--[[
+Holds configuration.
+A one-stop-shop for runtime configuration
+]]
 return {
 	domain = "test.monster:8888",
 	production = false,

src/lua/endpoints/claim_get.lua

@@ -2,12 +2,11 @@ local cache = require("cache")
 local config = require("config")
 local pages = require("pages")
 
-
 local function claim_get(req)
 	--Get the page to claim a name
 	local cachestr = string.format("%s/_claim",config.domain)
 	local text = cache.render(cachestr,function()
-		print("cache miss, rendering claim page")
+		log(LOG_DEBUG,"Cache miss, rendering claim page")
 		return pages.claim{err=""}
 	end)
 	http_response(req,200,text)

src/lua/endpoints/claim_post.lua

@@ -24,7 +24,7 @@ local function claim_post(req)
 	--What in the world, Kore should be rejecting names that
 	--are not lower case & no symbols, but some still get through somehow.
 	if not name:match("^[a-z0-9]*$") then
-		print("Bad username:",name)
+		log(LOG_DEBUG,"Bad username:" .. name)
 		text = pages.claim{
 			err = "Usernames must match ^[a-z0-9]{1,30}$"
 		}
@@ -44,7 +44,7 @@ local function claim_post(req)
 	stmnt_author_create:bind_blob(3,hash)
 	local err = util.do_sql(stmnt_author_create)
 	if err == sql.DONE then
-		print("success")
+		log(LOG_INFO,"Account creation successful:" .. name)
 		--We sucessfully made the new author
 		local id = stmnt_author_create:last_insert_rowid()
 		stmnt_author_create:reset()
@@ -53,7 +53,6 @@ local function claim_post(req)
 		http_response_header(req,"Content-Disposition","attachment; filename=\"" .. name .. "." .. config.domain .. ".passfile\"")
 		local session = sessionlib.start(id)
 		text = password
-		print("session started, about  to send password:",text)
 		http_response(req,200,text)
 		return
 	elseif err == sql.CONSTRAINT then
@@ -63,6 +62,7 @@ local function claim_post(req)
 			err = "Failed to claim. That name may already be taken."
 		}
 	elseif err == sql.ERROR or err == sql.MISUSE then
+		log(LOG_ALERT,"Account creation failed in an unusual way:" .. err)
 		--This is bad though
 		text = pages.claim {
 			err = "Failed to claim"

src/lua/endpoints/download_get.lua

@@ -18,6 +18,7 @@ local function download_get(req)
 	local path = http_request_get_path(req)
 	http_request_populate_qs(req)
 	local story = assert(http_argument_get_string(req,"story"))
+	story = util.decodeentities(story)
 	local story_id = util.decode_id(story)
 	stmnt_download:bind_names{
 		postid = story_id

src/lua/endpoints/edit_get.lua

@@ -26,7 +26,6 @@ local function edit_get(req)
 	local story_id = util.decode_id(story)
 	local ret
 
-	print("we want to edit story:",story)
 	--Check that the logged in user is the owner of the story
 	--sql-side. If we're not the owner, we'll get 0 rows back.
 	stmnt_edit:bind_names{

src/lua/endpoints/edit_post.lua

@@ -64,18 +64,6 @@ local function edit_post(req)
 	assert(util.do_sql(stmnt_update) == sql.DONE, "Failed to update text")
 	stmnt_update:reset()
 	tagslib.set(storyid,tags)
-	--[[
-	assert(stmnt_drop_tags:bind_names{postid = storyid} == sql.OK)
-	do_sql(stmnt_drop_tags)
-	stmnt_drop_tags:reset()
-	for _,tag in pairs(tags) do
-		print("Looking at tag",tag)
-		assert(stmnt_ins_tag:bind(1,storyid) == sql.OK)
-		assert(stmnt_ins_tag:bind(2,tag) == sql.OK)
-		err = do_sql(stmnt_ins_tag)
-		stmnt_ins_tag:reset()
-	end
-	]]
 	local id_enc = util.encode_id(storyid)
 	local loc = string.format("https://%s/%s",config.domain,id_enc)
 	cache.dirty(string.format("%s/%s",config.domain,id_enc)) -- This place to read this post

src/lua/endpoints/index_get.lua

@@ -22,25 +22,19 @@ function configure(...)
 end
 
 local function get_site_home(req)
-	print("Cache miss, rendering index")
+	log(LOG_DEBUG,"Cache miss, rendering site index")
 	stmnt_index:bind_names{}
-	local err = util.do_sql(stmnt_index)
 	local latest = {}
-	--err may be sql.ROW or sql.DONE if we don't have any stories yet
+	for tagsr, idr, title, iar, dater, author in util.sql_rows(stmnt_index) do
-	while err == sql.ROW do
-		local data = stmnt_index:get_values()
-		local storytags = libtags.get(data[1])
 		table.insert(latest,{
-			url = util.encode_id(data[1]),
+			url = util.encode_id(idr),
-			title = data[2],
+			title = title,
-			isanon = data[3] == 1,
+			isanon = tonumber(iar) == 1,
-			posted = os.date("%B %d %Y",tonumber(data[4])),
+			posted = os.date("%B %d %Y",tonumber(dater)),
-			author = data[5],
+			author = author,
-			tags = storytags,
+			tags = libtags.get(tagsr),
 		})
-		err = stmnt_index:step()
 	end
-	stmnt_index:reset()
 	return pages.index{
 		domain = config.domain,
 		stories = latest
@@ -52,6 +46,7 @@ local function get_author_home(req)
 	stmnt_author_bio:bind_names{author=subdomain}
 	local err = util.do_sql(stmnt_author_bio)
 	if err == sql.DONE then
+		log(LOG_INFO,"No such author:" .. subdomain)
 		stmnt_author_bio:reset()
 		return pages.noauthor{
 			author = subdomain
@@ -62,21 +57,15 @@ local function get_author_home(req)
 	local bio = data[1]
 	stmnt_author_bio:reset()
 	stmnt_author:bind_names{author=subdomain}
-	err = util.do_sql(stmnt_author)
 	local stories = {}
-	while err == sql.ROW do
+	for id, title, time in util.sql_rows(stmnt_author) do
-		local data = stmnt_author:get_values()
-		local id, title, time = unpack(data)
-		local tags = libtags.get(id)
 		table.insert(stories,{
 			url = util.encode_id(id),
 			title = title,
 			posted = os.date("%B %d %Y",tonumber(time)),
-			tags = tags,
+			tags = libtags.get(id),
 		})
-		err = stmnt_author:step()
 	end
-	stmnt_author:reset()
 	return pages.author_index{
 		domain=config.domain,
 		author=subdomain,
@@ -89,16 +78,16 @@ end
 local function index_get(req)
 	local method = http_method_text(req)
 	local host = http_request_get_host(req)
-	local path = http_request_get_path(req)
-	--Default home page
 	local subdomain = host:match("([^\\.]+)")
 	local text
 	if host == config.domain then
+		--Default home page
 		local cachepath = string.format("%s",config.domain)
 		text = cache.render(cachepath, function()
 			return get_site_home(req)
 		end)
-	else --author home page
+	else 
+		--author home page
 		local cachepath = string.format("%s.%s",subdomain,config.domain)
 		text = cache.render(cachepath, function()
 			return get_author_home(req)

src/lua/endpoints/paste_get.lua

@@ -14,7 +14,7 @@ local function paste_get(req)
 		return
 	elseif host == config.domain and author == nil then
 		text = cache.render(string.format("%s/_paste",host),function()
-			print("Cache missing, rendering post page")
+			log(LOG_DEBUG, "Cache missing, rendering post page")
 			return pages.paste{
 				domain = config.domain,
 				err = "",
@@ -41,56 +41,6 @@ local function paste_get(req)
 	end
 	assert(text)
 	http_response(req,200,text)
-	--[=[
-	if host == config.domain then
-		local author,_ = get_session(req)
-		if author then
-			http_response_header(req,"Location",string.format("https://%s.%s/_paste",author,domain))
-			http_response(req,303,"")
-			return
-		else
-			--For an anonymous user
-			ret = cache.render(string.format("%s/_paste",host),function()
-				print("Cache missing, rendering post page")
-				return pages.paste{
-					domain = domain,
-					err = "",
-				}
-			end)
-		end
-
-	else
-		--Or for someone that's logged in
-		print("Looks like a logged in user wants to paste!")
-		local subdomain = host:match("([^%.]+)")
-		local author,_ = session.get(req)
-		print("subdomain:",subdomain,"author:",author)
-		--If they try to paste as an author, but are on the
-		--wrong subdomain, or or not logged in, redirect them
-		--to the right place. Their own subdomain for authors
-		--or the anonymous paste page for not logged in users.
-		if author == nil then
-			http_response_header(req,"Location","https://"..domain.."/_paste")
-			http_response(req,303,"")
-			return
-		end
-		if author ~= subdomain then
-			http_response_header(req,"Location",string.format("https://%s.%s/_paste",author,domain))
-			http_response(req,303,"")
-			return
-		end
-		assert(author == subdomain,"someone wants to paste as someone else")
-		--We're where we want to be, serve up this users's
-		--paste page. No cache, because how often is a user
-		--going to paste?
-		ret = pages.author_paste{
-			domain = domain,
-			user = author,
-			text = "",
-			err = "",
-		}
-	end
-	]=]
 end
 
 return paste_get

src/lua/endpoints/paste_post.lua

@@ -35,15 +35,10 @@ local function anon_paste(req,ps)
 	--with a more elegent solution.
 
 	util.sqlbind(stmnt_paste,"bind_blob",1,ps.text)
-	--assert(stmnt_paste:bind_blob(1,text) == sql.OK)
 	util.sqlbind(stmnt_paste,"bind",2,ps.title)
-	--assert(stmnt_paste:bind(2,esctitle) == sql.OK)
 	util.sqlbind(stmnt_paste,"bind",3,-1)
-	--assert(stmnt_paste:bind(3,-1) == sql.OK)
 	util.sqlbind(stmnt_paste,"bind",4,true)
-	--assert(stmnt_paste:bind(4,true) == sql.OK)
 	util.sqlbind(stmnt_paste,"bind_blob",5,"")
-	--assert(stmnt_paste:bind_blob(5,"") == sql.OK)
 	err = util.do_sql(stmnt_paste)
 	stmnt_paste:reset()
 	if err == sql.DONE then
@@ -76,10 +71,10 @@ local function author_paste(req,ps)
 	local author, authorid = session.get(req)
 	if author == nil then
 		ret = pages.author_paste{
-			domain = domain,
+			domain = config.domain,
-			author = subdomain,
+			author = ps.subdomain,
-			err = "You are not logged in, you must be logged in to post as " .. subdomain .. ".",
+			err = "You are not logged in, you must be logged in to post as " .. ps.subdomain .. ".",
-			text = text
+			text = ps.text
 		}
 	end
 	local asanon = assert(http_argument_get_string(req,"pasteas"))
@@ -89,11 +84,7 @@ local function author_paste(req,ps)
 	assert(stmnt_paste:bind_blob(1,ps.text) == sql.OK)
 	assert(stmnt_paste:bind(2,ps.title) == sql.OK)
 	assert(stmnt_paste:bind(3,authorid) == sql.OK)
-	if asanon == "anonymous" then
+	assert(stmnt_paste:bind(4,asanon == "anonymous") == sql.OK)
-		assert(stmnt_paste:bind(4,true) == sql.OK)
-	else
-		assert(stmnt_paste:bind(4,false) == sql.OK)
-	end
 	assert(stmnt_paste:bind_blob(5,"") == sql.OK)
 	err = util.do_sql(stmnt_paste)
 	stmnt_paste:reset()
@@ -144,6 +135,7 @@ local function paste_post(req)
 	
 	local ps = {}
 	--We're creating a new paste
+	ps.subdomain = host:match("([^\\.]+)")
 	http_request_populate_post(req)
 	local title = assert(http_argument_get_string(req,"title"))
 	local text = assert(http_argument_get_string(req,"text"))
@@ -155,13 +147,13 @@ local function paste_post(req)
 	end
 	local pasteas
 	ps.raw = zlib.compress(text)
-	text = string.gsub(text,"%%(%x%x)",decodeentities)
+	text = util.decodeentities(text)
 	text = parsers[ps.markup](text)
 	assert(text,"Failed to parse text")
 	text = zlib.compress(text)
 	assert(text,"Failed to compress text")
 	ps.text = text
-	local esctitle = string.gsub(title,"%%(%x%x)",decodeentities)
+	local esctitle = util.decodeentities(title)
 	--Always sanatize the title with the plain parser. no markup
 	--in the title.
 	ps.title = parsers.plain(title)

src/lua/endpoints/preview_post.lua

@@ -5,7 +5,6 @@ local pages = require("pages")
 local config = require("config")
 
 local function preview_post(req)
-	print("We want to preview a paste!")
 	local host = http_request_get_host(req)
 	local path = http_request_get_path(req)
 	http_request_populate_post(req)
@@ -17,7 +16,6 @@ local function preview_post(req)
 	if tag_str then
 		tags = util.parse_tags(tag_str)
 	end
-	print("title:",title,"text:",text,"markup:",markup)
 	local parsed = parsers[markup](text)
 	local ret = pages.read{
 		domain = config.domain,

src/lua/endpoints/read_get.lua

@@ -45,7 +45,7 @@ local function populate_ps_story(req,ps)
 	if err == sql.DONE then
 		--We got no story
 		stmnt_read:reset()
-		print("No story by this name",ps.storyid)
+		log(LOG_DEBUG,"No story with id:" .. ps.storyid)
 		return false
 	end
 	--If we've made it here, we have a story. Populate our settings
@@ -73,41 +73,17 @@ local function get_comments(req,ps)
 	stmnt_comments:bind_names{
 		id = ps.storyid
 	}
-	err = util.do_sql(stmnt_comments)
 	local comments = {}
-	while err ~= sql.DONE do
+	for com_author, com_isanon, com_text in util.sql_rows(stmnt_comments) do
-		local com_author, com_isanon, com_text = unpack(stmnt_comments:get_values())
 		table.insert(comments,{
 			author = com_author,
 			isanon = com_isanon == 1, --int to boolean
 			text = com_text
 		})
-		err = stmnt_comments:step()
 	end
-	stmnt_comments:reset()
 	return comments
 end
 
---[[
-The author is viewing their own story, give them an edit button
-]]
-local function read_get_author(req,storyid,author,authorid,comments)
-
-end
-
---[[
-An author is viewing a story, allow them to post comments as themselves
-]]
-local function read_get_loggedin(req,ps)
-	if ps.tauthor == ps.authorid then
-		--The story exists and we're logged in as the
-		--owner, display the edit button
-		return read_get_author(req,ps)
-	end
-	return pages.read(ps)
-end
-
-
 local function read_get(req)
 	--Pages settings
 	local ps = {
@@ -116,7 +92,6 @@ local function read_get(req)
 		path = http_request_get_path(req),
 		method = http_method_text(req),
 	}
-	print("reading", ps.path)
 
 	--Get our story id
 	assert(string.len(ps.path) > 0,"Tried to read 0-length story id")
@@ -142,13 +117,13 @@ local function read_get(req)
 	local text
 	--normal story display
 	if (not ps.loggedauthor) then
-		print("not author")
 		local cachestr = string.format("%s%s%s",
 			ps.host,
 			ps.path,
 			ps.show_comments and "?comments=1" or ""
 		)
 		text = cache.render(cachestr,function()
+			log(LOG_DEBUG,"Cache miss, rendering story " .. cachestr)
 			if not populate_ps_story(req,ps) then
 				return pages.nostory(ps)
 			end
@@ -157,14 +132,13 @@ local function read_get(req)
 			return output
 		end)
 	else --we are logged in, don't cache
-		print("is author")
 		if not populate_ps_story(req,ps) then
-			return pages.nostory(ps)
+			text = pages.nostory(ps)
-		end
+		else
-		print("tauthor was", ps.tauthor, "while author was:",ps.author)
 			ps.owner = (ps.loggedauthorid == ps.tauthor)
 			text = pages.read(ps)
 		end
+	end
 	assert(text)
 	http_response(req,200,text)
 	return

src/lua/init.lua

@@ -1,4 +1,9 @@
-
+--[[
+Contains the code entrypoint, this is the first (and only) file run from kore.
+It registers a bunch of global functions that get called from kore when users
+visit particular pages. See src/smr.c for the names of the public functions.
+See conf/smr.conf for the data that can be access in each function
+]]
 print("Really fast print from init.lua")
 
 --Luarocks libraries
@@ -6,7 +11,7 @@ local et = require("etlua")
 local sql = require("lsqlite3")
 local zlib = require("zlib")
 
---stubs for overloading
+--stub for detouring
 function configure(...) end
 
 --smr code

src/lua/parser_imageboard.lua

@@ -1,3 +1,20 @@
+--[[
+A parser that approximates 8chan's markup:
+
+Surround text with double single-quotes(') to make text italic
+Surround text with triple single-quotes to make text bold
+Surround text with underscores(_) to make it underlined
+Surround text with double asterisks(*) to make it spoilered
+Surround text with tildes(~) to make it strike through
+Begin a line with a greater-than followed by a a space to make it
+>greentext
+Begin a line with a less-than followed by a space to make it
+<pinktext
+Surround text with forum-style [spoiler] and [/spoiler] tags as a second way to spoiler
+Surround text with forum-style [code] and [/code] tags to make it preformatted and monospace
+
+]]
+
 local lpeg = require("lpeg")
 lpeg.locale(lpeg)
 local V,P,C,S,B,Cs = lpeg.V,lpeg.P,lpeg.C,lpeg.S,lpeg.B,lpeg.Cs
@@ -45,7 +62,6 @@ local function tag(name,format)
 	local start_tag = P(string.format("[%s]",name))
 	local end_tag = P(string.format("[/%s]",name))
 	return start_tag * Cs(((1 - end_tag))^1) * end_tag / function(a)
-		print("sanatizing tag:",name,"data:",a)
 		return string.format(format,sanitize(a))
 	end
 end

src/lua/parser_plain.lua

@@ -1,3 +1,6 @@
+--[[
+A plain parser that dosen't do much
+]]
 --Characters to escape in the body text
 local escapes = {
 	["&"] = "&",

src/lua/session.lua

@@ -31,7 +31,6 @@ function session.get(req)
 	if err ~= sql.ROW then
 		return nil, "No such session by logged in users"
 	end
-	print("get session err:",err)
 	local data = stmnt_get_session:get_values()
 	stmnt_get_session:reset()
 	local author = data[1]
@@ -52,15 +51,12 @@ function session.start(who)
 	end
 	local session = table.concat(session_t)
 	rngf:close()
-	print("sessionid:",session)
-	print("authorid:",who)
 	stmnt_insert_session:bind_names{
 		sessionid = session,
 		authorid = who
 	}
 	local err = util.do_sql(stmnt_insert_session)
 	stmnt_insert_session:reset()
-	print("Err:",err)
 	assert(err == sql.DONE)
 	return session
 end

src/lua/tags.lua

@@ -45,14 +45,13 @@ function tags.set(storyid,tags)
 	stmnt_drop_tags:reset()
 	local err
 	for _,tag in pairs(tags) do
-		print("Looking at tag",tag)
 		assert(stmnt_ins_tag:bind(1,storyid) == sql.OK)
 		assert(stmnt_ins_tag:bind(2,tag) == sql.OK)
 		err = util.do_sql(stmnt_ins_tag)
 		stmnt_ins_tag:reset()
 	end
 	if err ~= sql.DONE then
-		print("Failed to save tags, but paste and raw still went through")
+		log(LOG_CRIT,"Failed to save tags on " .. storyid)
 	end
 
 end

src/lua/util.lua

@@ -24,7 +24,6 @@ function util.do_sql(stmnt)
 	local i = 0
 	repeat
 		err = stmnt:step()
-		print("After stepping, err is", err)
 		if err == sql.BUSY then
 			i = i + 1
 			coroutine.yield()
@@ -34,6 +33,36 @@ function util.do_sql(stmnt)
 	return err
 end
 
+--[[
+Provides an iterator that loops over results in an sql statement
+or throws an error, then resets the statement after the loop is done.
+]]
+function util.sql_rows(stmnt)
+	if not stmnt then error("No statement",2) end
+	local err
+	return function()
+		err = stmnt:step()
+		if err == sql.BUSY then
+			coroutine.yield()
+		elseif err == sql.ROW then
+			return unpack(stmnt:get_values())
+		elseif err == sql.DONE then
+			stmnt:reset()
+			return nil
+		else
+			stmnt:reset()
+			local msg = string.format(
+				"SQL Iteration failed: %s : %s\n%s",
+				tostring(err),
+				db.conn:errmsg(),
+				debug.traceback()
+			)
+			log(LOG_CRIT,msg)
+			error(msg)
+		end
+	end
+end
+
 --[[
 Binds an argument to as statement with nice error reporting on failure
 stmnt    :: sql.stmnt - the prepared sql statemnet
@@ -120,4 +149,17 @@ function util.parse_tags(str)
 	return tags
 end
 
+local function decodeentity(capture)
+	local n = tonumber(capture,16)
+	local c = string.char(n)
+	if escapes[c] then
+		return escapes[c]
+	else
+		return c
+	end
+end
+function util.decodeentities(str)
+	return string.gsub(str,"%%(%x%x)",decodeentity)
+end
+
 return util